From 07e265922ba623e661097f9ba286e2d010b6eeb0 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Tue, 26 May 2026 15:59:47 -0400 Subject: [PATCH 1/9] policy transform --- .github/workflows/tero-release-please.yaml | 44 ++ .github/workflows/tero-release.yaml | 358 +++++++++++ .release-please-manifest.json | 3 + Cargo.lock | 153 ++++- Cargo.toml | 4 + changelog.d/policy_transform.feature.md | 3 + distribution/docker/tero/Dockerfile | 50 ++ distribution/docker/tero/README.md | 84 +++ release-please-config.json | 28 + src/transforms/mod.rs | 2 + src/transforms/policy/adapter.rs | 636 +++++++++++++++++++ src/transforms/policy/config.rs | 160 +++++ src/transforms/policy/field_mapping.rs | 182 ++++++ src/transforms/policy/internal_events.rs | 53 ++ src/transforms/policy/mod.rs | 17 + src/transforms/policy/tests.rs | 677 +++++++++++++++++++++ src/transforms/policy/transform.rs | 105 ++++ 17 files changed, 2556 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/tero-release-please.yaml create mode 100644 .github/workflows/tero-release.yaml create mode 100644 .release-please-manifest.json create mode 100644 changelog.d/policy_transform.feature.md create mode 100644 distribution/docker/tero/Dockerfile create mode 100644 distribution/docker/tero/README.md create mode 100644 release-please-config.json create mode 100644 src/transforms/policy/adapter.rs create mode 100644 src/transforms/policy/config.rs create mode 100644 src/transforms/policy/field_mapping.rs create mode 100644 src/transforms/policy/internal_events.rs create mode 100644 src/transforms/policy/mod.rs create mode 100644 src/transforms/policy/tests.rs create mode 100644 src/transforms/policy/transform.rs diff --git a/.github/workflows/tero-release-please.yaml b/.github/workflows/tero-release-please.yaml new file mode 100644 index 0000000000000..4d7d19262e860 --- /dev/null +++ b/.github/workflows/tero-release-please.yaml @@ -0,0 +1,44 @@ +--- +# Release Please — Tero fork +# +# Watches `master` for commits using the Conventional Commits format and +# opens a "release PR" that bumps the prerelease version (.X-tero.N → .X-tero.N+1) +# and updates `Cargo.toml`, `CHANGELOG.md`, and `.release-please-manifest.json`. +# Merging that PR creates the `v` tag, which triggers +# `tero-release.yaml` to publish artifacts. +# +# This is the Tero-fork equivalent of `tero-collector-distro`'s +# `release-please.yaml`. Do not touch upstream Vector's `release.yml`. + +name: Tero release please + +on: + push: + branches: + - master + +permissions: + contents: write + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + release-please: + runs-on: ubuntu-24.04 + # Don't fire in the upstream `vectordotdev/vector` repo if this fork's + # `master` is ever pushed back upstream by accident. + if: github.repository_owner == 'usetero' + steps: + - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 + with: + # GITHUB_TOKEN works, but PRs it creates cannot trigger other + # workflows. If you want `tero-release.yaml` to fire when the + # release PR is merged (and the tag is pushed automatically), swap + # this for a PAT or GitHub App token. Today the tag push DOES + # trigger workflows because release-please pushes it directly. + token: ${{ secrets.GITHUB_TOKEN }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json diff --git a/.github/workflows/tero-release.yaml b/.github/workflows/tero-release.yaml new file mode 100644 index 0000000000000..4c687dfb26d22 --- /dev/null +++ b/.github/workflows/tero-release.yaml @@ -0,0 +1,358 @@ +--- +# Tero release pipeline. +# +# Triggers on `v*` tags created by `tero-release-please.yaml` (or `workflow_dispatch` +# for a smoke test). Builds the Vector binary natively on amd64 + arm64 +# Linux runners, produces a tarball per arch, then bakes both into a +# multi-arch Docker image at `ghcr.io/${REPO}:VERSION`, and finally attaches +# the tarballs to the GitHub Release that release-please already created. +# +# This pipeline is intentionally narrow — no .deb / .rpm / .msi / macOS / +# Windows / musl / armv7. The upstream `release.yml` / `publish.yml` +# pipelines that produce those artifacts remain untouched but should be +# disabled in repository Settings → Actions → Workflows for this fork +# (see distribution/docker/tero/README.md). + +name: Tero release + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: "Version to release (without v prefix). Use for dry runs." + required: false + default: "0.0.0-tero.dryrun" + dry_run: + description: "Build but do not push images or attach release assets." + required: false + default: false + type: boolean + +permissions: + contents: write + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + CARGO_TERM_COLOR: always + RUSTFLAGS: "-C strip=symbols" + +jobs: + prepare: + name: Prepare release + runs-on: ubuntu-24.04 + if: github.repository_owner == 'usetero' + outputs: + version: ${{ steps.version.outputs.version }} + tag: ${{ steps.version.outputs.tag }} + steps: + - name: Determine version + id: version + env: + INPUT_VERSION: ${{ inputs.version }} + REF: ${{ github.ref }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ "${EVENT_NAME}" = "workflow_dispatch" ]; then + VERSION="${INPUT_VERSION}" + else + VERSION="${REF#refs/tags/v}" + fi + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" + echo "tag=v${VERSION}" >> "${GITHUB_OUTPUT}" + echo "Resolved version: ${VERSION}" + + build-binary: + name: Build binary (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + needs: prepare + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + triple: x86_64-unknown-linux-gnu + - arch: arm64 + runner: ubuntu-24.04-arm + triple: aarch64-unknown-linux-gnu + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install build dependencies + # Vectorscan (via `vectorscan-rs-sys`) needs cmake + a C++ toolchain + # + ragel + libpcre2. Everything else (libsasl2, libssl, krb5) is + # required by Vector's default `target-*` feature set. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + libclang-dev \ + libpcre2-dev \ + libsasl2-dev \ + libssl-dev \ + libxml2-dev \ + pkg-config \ + ragel \ + unzip + # `target-*-linux-gnu` pulls in `vendored` which builds krb5/gssapi + # from source; install its build-time prereqs too. + sudo apt-get install -y --no-install-recommends \ + bison \ + comerr-dev \ + libkrb5-dev + + - name: Install Rust toolchain + # rust-toolchain.toml pins the channel — the action respects it. + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "" + + - name: Cache cargo registry & build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: tero-release-${{ matrix.triple }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + tero-release-${{ matrix.triple }}- + + - name: Build Vector + env: + TRIPLE: ${{ matrix.triple }} + run: | + cargo build \ + --release \ + --target "${TRIPLE}" \ + --no-default-features \ + --features "target-${TRIPLE},transforms-policy" + + - name: Stage binary artifacts + env: + VERSION: ${{ needs.prepare.outputs.version }} + TRIPLE: ${{ matrix.triple }} + run: | + ARCHIVE_DIR="vector-${VERSION}-${TRIPLE}" + mkdir -p "${ARCHIVE_DIR}" + cp "target/${TRIPLE}/release/vector" "${ARCHIVE_DIR}/vector" + cp LICENSE "${ARCHIVE_DIR}/LICENSE" + cp README.md "${ARCHIVE_DIR}/README.md" + chmod +x "${ARCHIVE_DIR}/vector" + tar -czf "${ARCHIVE_DIR}.tar.gz" "${ARCHIVE_DIR}" + sha256sum "${ARCHIVE_DIR}.tar.gz" > "${ARCHIVE_DIR}.tar.gz.sha256" + ls -lh "${ARCHIVE_DIR}.tar.gz" + + - name: Verify binary + env: + TRIPLE: ${{ matrix.triple }} + run: "./target/${TRIPLE}/release/vector --version" + + - name: Upload tarball artifact + uses: actions/upload-artifact@v4 + with: + name: vector-binary-${{ matrix.arch }} + path: | + vector-*.tar.gz + vector-*.tar.gz.sha256 + if-no-files-found: error + retention-days: 1 + + - name: Upload bare binary for Docker stage + uses: actions/upload-artifact@v4 + with: + name: vector-bin-${{ matrix.arch }} + path: target/${{ matrix.triple }}/release/vector + if-no-files-found: error + retention-days: 1 + + build-docker: + name: Build Docker image (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + needs: [prepare, build-binary] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + platform: linux/amd64 + - arch: arm64 + runner: ubuntu-24.04-arm + platform: linux/arm64 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download binary for this arch + uses: actions/download-artifact@v4 + with: + name: vector-bin-${{ matrix.arch }} + path: ${{ matrix.arch }}/ + + - name: Assemble build context + run: | + chmod +x "${{ matrix.arch }}/vector" + ls -la "${{ matrix.arch }}/" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: ${{ !inputs.dry_run }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: distribution/docker/tero/Dockerfile + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ !inputs.dry_run }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + + - name: Export digest + if: ${{ !inputs.dry_run }} + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + mkdir -p /tmp/digests + touch "/tmp/digests/${DIGEST#sha256:}" + + - name: Upload digest + if: ${{ !inputs.dry_run }} + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge-docker-manifest: + name: Merge Docker manifests + runs-on: ubuntu-24.04 + needs: [prepare, build-docker] + if: ${{ !inputs.dry_run }} + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ needs.prepare.outputs.version }} + type=raw,value=latest,enable=${{ github.event_name == 'push' }} + type=sha + + - name: Create multi-arch manifest and push + working-directory: /tmp/digests + env: + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + run: | + # shellcheck disable=SC2046 + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "${DOCKER_METADATA_OUTPUT_JSON}") \ + $(printf "${IMAGE}@sha256:%s " *) + + - name: Inspect image + env: + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.version }} + run: docker buildx imagetools inspect "${IMAGE}" + + upload-release-assets: + name: Attach binaries to GitHub Release + runs-on: ubuntu-24.04 + needs: [prepare, build-binary, merge-docker-manifest] + if: ${{ !inputs.dry_run && github.event_name == 'push' }} + permissions: + contents: write + steps: + - name: Download binary tarballs + uses: actions/download-artifact@v4 + with: + path: /tmp/assets + pattern: vector-binary-* + merge-multiple: true + + - name: List assets + run: ls -la /tmp/assets + + - name: Upload to GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ needs.prepare.outputs.tag }} + # `gh release upload` requires the release to already exist. release-please + # creates it at tag time; this job runs immediately after, so the release + # should be there. `--clobber` lets us re-run safely. + run: | + cd /tmp/assets + gh release upload "${TAG}" \ + --repo "${{ github.repository }}" \ + --clobber \ + vector-*.tar.gz \ + vector-*.tar.gz.sha256 + + summary: + name: Release summary + runs-on: ubuntu-24.04 + needs: [prepare, build-binary, build-docker, merge-docker-manifest, upload-release-assets] + if: always() + steps: + - name: Summarize + env: + VERSION: ${{ needs.prepare.outputs.version }} + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + { + echo "## Tero Vector release" + echo + echo "Version: \`${VERSION}\`" + echo + if [ "${DRY_RUN}" = "true" ]; then + echo "**Dry run.** Artifacts built; nothing pushed." + else + echo "Docker image:" + echo + echo " docker pull ${IMAGE}:${VERSION}" + echo + echo "Binary tarballs:" + echo + echo "* \`vector-${VERSION}-x86_64-unknown-linux-gnu.tar.gz\`" + echo "* \`vector-${VERSION}-aarch64-unknown-linux-gnu.tar.gz\`" + fi + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000000000..cbd43ad2679a3 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.56.0-tero.0" +} diff --git a/Cargo.lock b/Cargo.lock index f8841d4be18e1..26ee2a657b706 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4365,6 +4365,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -5944,6 +5954,17 @@ dependencies = [ "snafu 0.7.5", ] +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + [[package]] name = "inotify" version = "0.11.0" @@ -5984,6 +6005,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -7500,6 +7530,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" +[[package]] +name = "notify" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +dependencies = [ + "bitflags 2.10.0", + "filetime", + "fsevent-sys", + "inotify 0.10.2", + "kqueue", + "libc", + "log", + "mio", + "notify-types 1.0.1", + "walkdir", + "windows-sys 0.52.0", +] + [[package]] name = "notify" version = "8.2.0" @@ -7508,16 +7557,25 @@ checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ "bitflags 2.10.0", "fsevent-sys", - "inotify", + "inotify 0.11.0", "kqueue", "libc", "log", "mio", - "notify-types", + "notify-types 2.0.0", "walkdir", "windows-sys 0.60.2", ] +[[package]] +name = "notify-types" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585d3cb5e12e01aed9e8a1f70d5c6b5e86fe2a6e48fc8cd0b3e0b8df6f6eb174" +dependencies = [ + "instant", +] + [[package]] name = "notify-types" version = "2.0.0" @@ -8147,6 +8205,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5a797f0e07bdf071d15742978fc3128ec6c22891c31a3a931513263904c982a" +[[package]] +name = "pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +dependencies = [ + "base64 0.21.7", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" +dependencies = [ + "heck 0.5.0", + "itertools 0.13.0", + "prost 0.13.5", + "prost-types 0.13.5", +] + [[package]] name = "pbkdf2" version = "0.11.0" @@ -8390,6 +8470,26 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "policy-rs" +version = "1.6.0" +source = "git+https://github.com/usetero/policy-rs?rev=fc94f17312aa51019276c97f7fb57abf20089bf5#fc94f17312aa51019276c97f7fb57abf20089bf5" +dependencies = [ + "fnv", + "notify 7.0.0", + "pbjson", + "pbjson-build", + "prost 0.13.5", + "prost-build 0.13.5", + "rand 0.8.5", + "regex", + "serde", + "serde_json", + "tokio", + "tonic-build 0.12.3", + "vectorscan-rs-sys", +] + [[package]] name = "polling" version = "3.7.4" @@ -11577,6 +11677,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tcp-stream" version = "0.34.2" @@ -12251,6 +12362,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2 1.0.106", + "prost-build 0.13.5", + "prost-types 0.13.5", + "quote 1.0.45", + "syn 2.0.117", +] + [[package]] name = "tonic-build" version = "0.13.1" @@ -13127,7 +13252,7 @@ dependencies = [ "nix 0.31.2", "nkeys", "nom 8.0.0", - "notify", + "notify 8.2.0", "opendal", "openssl", "openssl-probe", @@ -13137,6 +13262,7 @@ dependencies = [ "pastey", "percent-encoding", "pin-project", + "policy-rs", "postgres-openssl", "procfs", "proptest", @@ -13647,6 +13773,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "vectorscan-rs-sys" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473f4b86c4c9e42140712b9790079bca554f6341c8ce19f0f1a3ad3b9509dbd5" +dependencies = [ + "cmake", + "flate2", + "tar", +] + [[package]] name = "version_check" version = "0.9.4" @@ -14802,6 +14939,16 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 3290b7192da7d..133f31772db0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,6 +338,9 @@ prost-types = { workspace = true, optional = true } # Databricks Zerobus databricks-zerobus-ingest-sdk = { git = "https://github.com/databricks/zerobus-sdk-rs", rev = "273efe09130a7554e6b34d20b95df50289e58524", optional = true } +# Tero Policy (policy-rs) — used by the `policy` transform. +policy-rs = { git = "https://github.com/usetero/policy-rs", rev = "fc94f17312aa51019276c97f7fb57abf20089bf5", optional = true } + # GCP goauth = { version = "0.16.0", optional = true } smpl_jwt = { version = "0.8.0", default-features = false, optional = true } @@ -825,6 +828,7 @@ transforms-window = [] transforms-log_to_metric = [] transforms-lua = ["dep:mlua", "vector-lib/lua"] transforms-metric_to_log = [] +transforms-policy = ["dep:policy-rs"] transforms-reduce = ["transforms-impl-reduce"] transforms-remap = [] transforms-route = [] diff --git a/changelog.d/policy_transform.feature.md b/changelog.d/policy_transform.feature.md new file mode 100644 index 0000000000000..4c0df1ef9a11d --- /dev/null +++ b/changelog.d/policy_transform.feature.md @@ -0,0 +1,3 @@ +Added a new `policy` transform that evaluates log events against a JSON policy file using the [`policy-rs`](https://github.com/usetero/policy-rs) engine. The transform can keep, drop, sample, and rate-limit logs, and can apply field-level transformations (remove, redact, rename, add) based on the matched policy. Policies are watched and reloaded on file change. The transform is gated behind the `transforms-policy` cargo feature. + +authors: jaronoff97 diff --git a/distribution/docker/tero/Dockerfile b/distribution/docker/tero/Dockerfile new file mode 100644 index 0000000000000..76606ff266dd8 --- /dev/null +++ b/distribution/docker/tero/Dockerfile @@ -0,0 +1,50 @@ +# Tero Vector — runtime image +# +# Receives a prebuilt `vector` binary in the build context and bakes it into +# a minimal Debian-slim runtime. Build orchestration lives in +# `.github/workflows/tero-release.yaml`, which compiles the binary natively +# on amd64 and arm64 runners and supplies the right binary per --platform. +# +# Build args: +# TARGETARCH Set automatically by buildx (`amd64` | `arm64`). Selects +# which prebuilt binary directory to copy from. +# +# Build context layout (relative to context root): +# amd64/vector Linux x86_64 binary +# arm64/vector Linux aarch64 binary +# LICENSE License file (copied for OCI metadata) +# README.md Optional, included for OCI metadata + +FROM docker.io/library/debian:bookworm-slim@sha256:0104b334637a5f19aa9c983a91b54c89887c0984081f2068983107a6f6c21eeb AS runtime + +ARG TARGETARCH + +# Common Vector runtime dependencies plus libstdc++ for Vectorscan. +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + tzdata \ + libsasl2-2 \ + libssl3 \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* + +LABEL org.opencontainers.image.title="vector" +LABEL org.opencontainers.image.description="Tero distribution of Vector with the policy transform enabled." +LABEL org.opencontainers.image.source="https://github.com/usetero/vector" +LABEL org.opencontainers.image.licenses="MPL-2.0" +LABEL org.opencontainers.image.vendor="Tero" + +# Pull the per-arch binary from the build context. The workflow places +# binaries under `amd64/vector` and `arm64/vector` before invoking buildx. +COPY --chmod=0755 ${TARGETARCH}/vector /usr/bin/vector +COPY LICENSE /usr/share/doc/vector/LICENSE + +# Smoke test — fail the image build if the binary can't even print its version. +RUN ["vector", "--version"] + +# Vector's API listens on 8686 by default. +EXPOSE 8686 + +ENTRYPOINT ["/usr/bin/vector"] diff --git a/distribution/docker/tero/README.md b/distribution/docker/tero/README.md new file mode 100644 index 0000000000000..4b7041ddfe8a6 --- /dev/null +++ b/distribution/docker/tero/README.md @@ -0,0 +1,84 @@ +# Tero Vector distribution + +This directory holds the release machinery for the Tero fork of Vector. +The fork tracks upstream `vectordotdev/vector` and adds the `policy` +transform (see `src/transforms/policy/`); everything else is the +unmodified upstream tree. + +## What's produced + +Each release publishes: + +- A multi-arch Docker image to **`ghcr.io/usetero/vector`** at + `:` and `:latest` covering `linux/amd64` and `linux/arm64`. +- Two binary tarballs attached to the GitHub Release: + - `vector--x86_64-unknown-linux-gnu.tar.gz` + - `vector--aarch64-unknown-linux-gnu.tar.gz` +- A `.sha256` checksum next to each tarball. + +The binaries are built with `--features +"target-,transforms-policy"`, i.e. upstream Vector's standard +feature set for the target triple **plus** our `policy` transform. + +## How a release is cut + +1. Land PRs to `master` using Conventional Commits. `feat:`, `fix:`, + `chore:`, etc. +2. `tero-release-please.yaml` watches `master`. After each merge it opens + (or updates) a single "release PR" that bumps + `.release-please-manifest.json`, `Cargo.toml` version, and + `CHANGELOG.md`. +3. When the release PR is merged, release-please creates the + `v` tag and the corresponding GitHub Release. +4. The tag push triggers `tero-release.yaml`, which builds the binaries + and image and attaches the tarballs to the release. + +## Versioning + +Versions follow `-tero.` (e.g. `0.56.0-tero.1`). The base +`` should match the upstream Vector version this fork is +rebased on. `` is bumped automatically by release-please each release. + +When rebasing onto a newer upstream Vector: + +1. Rebase `master` onto the new upstream tag. +2. Manually edit `.release-please-manifest.json` to reset the base, e.g. + `0.56.0-tero.5` → `0.57.0-tero.0`. +3. Merge a `chore(release): rebase onto upstream vX.Y.Z` commit; the + next release-please PR bumps to `0.57.0-tero.1`. + +## One-time setup: disable upstream release workflows + +Upstream's `release.yml` and `publish.yml` also fire on `v*` tag pushes +and try to publish to channels we don't own (Docker Hub `timberio/vector`, +S3 `packages.timber.io`, Homebrew, the upstream GHCR). They will fail +loudly without those credentials. + +In the fork repository: **Settings → Actions → Workflows → disable**: + +- `Release` (`.github/workflows/release.yml`) +- `Publish` (`.github/workflows/publish.yml`) + +This is settings-only and avoids editing any upstream file. The +disablement persists across rebases. + +If you ever need the upstream pipelines back (e.g. to mirror Vector's +own release artifacts), re-enable them from the same panel. + +## Manual smoke test + +Trigger `tero-release.yaml` from the Actions tab with `workflow_dispatch`: + +- `version`: `0.0.0-tero.dryrun` (or any string) +- `dry_run`: `true` + +This builds the binaries and Docker image without pushing to GHCR or +attaching release assets. + +## Files in this directory + +- `Dockerfile` — Debian-slim runtime that receives a prebuilt `vector` + binary via the build context and bakes it into a multi-arch image. +- `README.md` — this file. + +Build orchestration lives in `.github/workflows/tero-release.yaml`. diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000000000..39c817b1e0e07 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "packages": { + ".": { + "release-type": "rust", + "prerelease": true, + "bump-minor-pre-major": false, + "bump-patch-for-minor-pre-major": false, + "draft": false, + "prerelease-type": "tero", + "include-component-in-tag": false, + "changelog-sections": [ + { "type": "feat", "section": "Features" }, + { "type": "fix", "section": "Bug Fixes" }, + { "type": "enhancement", "section": "Enhancements" }, + { "type": "perf", "section": "Performance" }, + { "type": "deps", "section": "Dependencies" }, + { "type": "revert", "section": "Reverts" }, + { "type": "docs", "section": "Documentation", "hidden": true }, + { "type": "chore", "section": "Miscellaneous", "hidden": true }, + { "type": "refactor", "section": "Code Refactoring", "hidden": true }, + { "type": "test", "section": "Tests", "hidden": true }, + { "type": "build", "section": "Build System", "hidden": true }, + { "type": "ci", "section": "Continuous Integration", "hidden": true } + ] + } + } +} diff --git a/src/transforms/mod.rs b/src/transforms/mod.rs index e4f6671828807..e34a98317322a 100644 --- a/src/transforms/mod.rs +++ b/src/transforms/mod.rs @@ -23,6 +23,8 @@ pub mod log_to_metric; pub mod lua; #[cfg(feature = "transforms-metric_to_log")] pub mod metric_to_log; +#[cfg(feature = "transforms-policy")] +pub mod policy; #[cfg(feature = "transforms-remap")] pub mod remap; #[cfg(feature = "transforms-route")] diff --git a/src/transforms/policy/adapter.rs b/src/transforms/policy/adapter.rs new file mode 100644 index 0000000000000..950a845b58ed1 --- /dev/null +++ b/src/transforms/policy/adapter.rs @@ -0,0 +1,636 @@ +//! Adapter that exposes a Vector `LogEvent` to the `policy-rs` engine. +//! +//! `policy-rs` evaluates against types that implement its `Matchable` and +//! `Transformable` traits. Vector's `LogEvent` is schema-less, so we wrap a +//! mutable borrow alongside a [`FieldMapping`] and translate every +//! `LogFieldSelector` into an event path on the fly. +//! +//! The adapter is intentionally side-effect free outside the borrow it +//! holds: every read/write goes through `LogEvent`'s own API, so future +//! `policy-rs` versions can change their evaluation strategy without +//! requiring changes anywhere else in Vector. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::LogField; +use policy_rs::{LogFieldSelector, Matchable, Transformable, engine::LogSignal}; +use vector_lib::{ + event::{LogEvent, Value}, + lookup::{OwnedTargetPath, OwnedValuePath, lookup_v2::ConfigValuePath}, +}; + +use super::field_mapping::FieldMapping; + +/// Wraps a mutable `LogEvent` and a `FieldMapping` so the `policy-rs` engine +/// can read and mutate it through `Matchable` and `Transformable`. +pub struct VectorLogAdapter<'a> { + log: &'a mut LogEvent, + mapping: &'a FieldMapping, +} + +impl<'a> VectorLogAdapter<'a> { + pub const fn new(log: &'a mut LogEvent, mapping: &'a FieldMapping) -> Self { + Self { log, mapping } + } + + /// Resolve a `LogFieldSelector` to the corresponding `LogEvent` path. + /// + /// Returns `None` for selectors that the current mapping does not + /// represent — for example, the protobuf-default `LogField::Unspecified` + /// variant or fields that haven't been wired up yet. The engine treats + /// `None` as "field not present", which is the correct fail-soft + /// behaviour for matching. + fn path_for(&self, selector: &LogFieldSelector) -> Option { + match selector { + LogFieldSelector::Simple(field) => self.simple_path(*field), + LogFieldSelector::LogAttribute(path) => Some(FieldMapping::append_segments( + &self.mapping.log_attributes, + path, + )), + LogFieldSelector::ResourceAttribute(path) => Some(FieldMapping::append_segments( + &self.mapping.resource_attributes, + path, + )), + LogFieldSelector::ScopeAttribute(path) => Some(FieldMapping::append_segments( + &self.mapping.scope_attributes, + path, + )), + } + } + + fn simple_path(&self, field: LogField) -> Option { + let mapped: &ConfigValuePath = match field { + LogField::Body => &self.mapping.body, + LogField::SeverityText => &self.mapping.severity_text, + LogField::TraceId => &self.mapping.trace_id, + LogField::SpanId => &self.mapping.span_id, + LogField::EventName => &self.mapping.event_name, + LogField::ResourceSchemaUrl => &self.mapping.resource_schema_url, + LogField::ScopeSchemaUrl => &self.mapping.scope_schema_url, + LogField::Unspecified => return None, + }; + Some(mapped.0.clone()) + } +} + +impl Matchable for VectorLogAdapter<'_> { + type Signal = LogSignal; + + fn get_field(&self, field: &LogFieldSelector) -> Option> { + let path = self.path_for(field)?; + let target = OwnedTargetPath::event(path); + let value = self.log.get(&target)?; + value_to_match_string(value) + } + + fn field_exists(&self, field: &LogFieldSelector) -> bool { + let Some(path) = self.path_for(field) else { + return false; + }; + let target = OwnedTargetPath::event(path); + self.log.get(&target).is_some() + } +} + +impl Transformable for VectorLogAdapter<'_> { + fn set_field(&mut self, field: &LogFieldSelector, value: &str) { + if let Some(path) = self.path_for(field) { + let target = OwnedTargetPath::event(path); + self.log.insert(&target, value.to_string()); + } + } + + fn delete_field(&mut self, field: &LogFieldSelector) -> bool { + let Some(path) = self.path_for(field) else { + return false; + }; + let target = OwnedTargetPath::event(path); + self.log.remove(&target).is_some() + } + + fn move_field(&mut self, from: &LogFieldSelector, to: &LogFieldSelector) { + // The engine guarantees `from` exists and `to` does not, so we can + // remove from the source and unconditionally insert at the target + // without losing data on a name collision. + let (Some(from_path), Some(to_path)) = (self.path_for(from), self.path_for(to)) else { + return; + }; + let from_target = OwnedTargetPath::event(from_path); + let to_target = OwnedTargetPath::event(to_path); + if let Some(value) = self.log.remove(&from_target) { + self.log.insert(&to_target, value); + } + } +} + +/// Coerce a `LogEvent` value to a string suitable for `policy-rs` pattern +/// matching. +/// +/// Strings (`Bytes` containing valid UTF-8) are borrowed; scalars are +/// stringified. Containers, timestamps, nulls, and regexes can't drive a +/// string match cleanly and return `None` — `field_exists` still reports +/// them as present so `exists: true` matchers fire correctly. +fn value_to_match_string(value: &Value) -> Option> { + match value { + Value::Bytes(bytes) => std::str::from_utf8(bytes).ok().map(Cow::Borrowed), + Value::Integer(i) => Some(Cow::Owned(i.to_string())), + Value::Float(f) => Some(Cow::Owned(f.to_string())), + Value::Boolean(b) => Some(Cow::Borrowed(if *b { "true" } else { "false" })), + Value::Timestamp(_) + | Value::Object(_) + | Value::Array(_) + | Value::Regex(_) + | Value::Null => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use policy_rs::proto::tero::policy::v1::LogField; + use vector_lib::event::LogEvent; + + fn make_log() -> LogEvent { + let mut log = LogEvent::default(); + log.insert("message", "hello world"); + log.insert("severity_text", "ERROR"); + log.insert("trace_id", "abc123"); + log.insert("span_id", "def456"); + log.insert("event_name", "user.login"); + log.insert( + "resource.schema_url", + "https://opentelemetry.io/schemas/1.0", + ); + log.insert("scope.schema_url", "https://opentelemetry.io/schemas/1.0"); + log.insert("attributes.user_id", "42"); + log.insert("attributes.flagged", true); + log.insert("attributes.ratio", 1.5); + log.insert("resource.attributes.service\\.name", "frontend"); + log.insert("scope.attributes.library", "tracer"); + log + } + + fn mapping() -> FieldMapping { + FieldMapping::default() + } + + // --- Matchable: get_field -------------------------------------------- + + #[test] + fn get_simple_body() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::Body)) + .as_deref(), + Some("hello world"), + ); + } + + #[test] + fn get_simple_severity_text() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::SeverityText)) + .as_deref(), + Some("ERROR"), + ); + } + + #[test] + fn get_simple_trace_and_span_ids() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::TraceId)) + .as_deref(), + Some("abc123"), + ); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::SpanId)) + .as_deref(), + Some("def456"), + ); + } + + #[test] + fn get_unspecified_field_returns_none() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter.get_field(&LogFieldSelector::Simple(LogField::Unspecified)), + None, + ); + } + + #[test] + fn get_log_attribute_flat() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .as_deref(), + Some("42"), + ); + } + + #[test] + fn get_log_attribute_nested() { + let mut log = LogEvent::default(); + log.insert("attributes.http.method", "GET"); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec![ + "http".to_string(), + "method".to_string() + ])) + .as_deref(), + Some("GET"), + ); + } + + #[test] + fn get_resource_attribute_with_dot_in_key() { + // policy-rs surfaces OTel attributes as single-segment selectors + // even when the OTel key contains dots (e.g. "service.name"). The + // adapter must treat each segment literally — segments containing + // dots are not re-parsed as dotted paths. + // + // Insert through a typed path that mirrors what `append_segments` + // produces, then confirm the adapter can read it back through the + // same selector. + let mut log = LogEvent::default(); + let m = mapping(); + let attr_path = + FieldMapping::append_segments(&m.resource_attributes, &["service.name".to_string()]); + log.insert(&OwnedTargetPath::event(attr_path), "frontend"); + + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::ResourceAttribute(vec![ + "service.name".to_string() + ])) + .as_deref(), + Some("frontend"), + ); + } + + #[test] + fn get_scope_attribute() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::ScopeAttribute(vec![ + "library".to_string() + ])) + .as_deref(), + Some("tracer"), + ); + } + + #[test] + fn get_integer_value_stringifies() { + let mut log = LogEvent::default(); + log.insert("attributes.count", 42_i64); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["count".to_string()])) + .as_deref(), + Some("42"), + ); + } + + #[test] + fn get_boolean_value_stringifies() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["flagged".to_string()])) + .as_deref(), + Some("true"), + ); + } + + #[test] + fn get_float_value_stringifies() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + let v = adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["ratio".to_string()])) + .map(Cow::into_owned); + // We don't pin the exact float representation — just confirm it's a string starting with "1.5". + assert!( + matches!(v, Some(ref s) if s.starts_with("1.5")), + "got: {v:?}" + ); + } + + #[test] + fn get_timestamp_returns_none_for_matching() { + let mut log = LogEvent::default(); + log.insert("attributes.ts", Value::Timestamp(Utc::now())); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter.get_field(&LogFieldSelector::LogAttribute(vec!["ts".to_string()])), + None, + ); + } + + #[test] + fn get_missing_field_returns_none() { + let mut log = LogEvent::default(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter.get_field(&LogFieldSelector::Simple(LogField::Body)), + None, + ); + } + + // --- Matchable: field_exists ----------------------------------------- + + #[test] + fn exists_true_for_string_field() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); + } + + #[test] + fn exists_true_for_non_string_field() { + // OTel-style: integer/bool/timestamp values should still satisfy + // `exists: true` matchers even though `get_field` returns None. + let mut log = LogEvent::default(); + log.insert("attributes.count", 42_i64); + log.insert("attributes.ts", Value::Timestamp(Utc::now())); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["count".to_string()]))); + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["ts".to_string()]))); + } + + #[test] + fn exists_false_for_missing_field() { + let mut log = LogEvent::default(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); + assert!( + !adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["missing".to_string()])) + ); + } + + #[test] + fn exists_false_for_unspecified() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Unspecified))); + } + + // --- Transformable: set_field ---------------------------------------- + + #[test] + fn set_field_creates_when_absent() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::LogAttribute(vec!["new_key".to_string()]), + "value", + ); + } + assert_eq!( + log.get("attributes.new_key").and_then(|v| v.as_str()), + Some("value".into()), + ); + } + + #[test] + fn set_field_overwrites_when_present() { + let mut log = make_log(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field(&LogFieldSelector::Simple(LogField::Body), "replaced"); + } + assert_eq!( + log.get("message").and_then(|v| v.as_str()), + Some("replaced".into()), + ); + } + + #[test] + fn set_nested_attribute_creates_intermediate_maps() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::LogAttribute(vec!["http".to_string(), "method".to_string()]), + "POST", + ); + } + assert_eq!( + log.get("attributes.http.method").and_then(|v| v.as_str()), + Some("POST".into()), + ); + } + + #[test] + fn set_unspecified_is_noop() { + let mut log = make_log(); + let snapshot_before = log.clone(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field(&LogFieldSelector::Simple(LogField::Unspecified), "x"); + } + assert_eq!(log, snapshot_before); + } + + // --- Transformable: delete_field ------------------------------------- + + #[test] + fn delete_field_removes_present_field() { + let mut log = make_log(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + }; + assert!(was_present); + assert!(log.get("attributes.user_id").is_none()); + } + + #[test] + fn delete_field_returns_false_when_absent() { + let mut log = LogEvent::default(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["nope".to_string()])) + }; + assert!(!was_present); + } + + #[test] + fn delete_field_returns_false_for_unspecified() { + let mut log = make_log(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::Simple(LogField::Unspecified)) + }; + assert!(!was_present); + } + + #[test] + fn delete_simple_body() { + let mut log = make_log(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::Simple(LogField::Body)) + }; + assert!(was_present); + assert!(log.get("message").is_none()); + } + + // --- Transformable: move_field --------------------------------------- + + #[test] + fn move_field_within_log_attributes() { + let mut log = LogEvent::default(); + log.insert("attributes.usr", "admin"); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["usr".to_string()]), + &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), + ); + } + assert!(log.get("attributes.usr").is_none()); + assert_eq!( + log.get("attributes.user_id").and_then(|v| v.as_str()), + Some("admin".into()), + ); + } + + #[test] + fn move_field_preserves_non_string_value() { + let mut log = LogEvent::default(); + log.insert("attributes.count", 42_i64); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["count".to_string()]), + &LogFieldSelector::LogAttribute(vec!["renamed".to_string()]), + ); + } + assert!(log.get("attributes.count").is_none()); + assert_eq!( + log.get("attributes.renamed").and_then(|v| v.as_integer()), + Some(42), + ); + } + + #[test] + fn move_field_absent_source_is_noop() { + let mut log = LogEvent::default(); + let snapshot_before = log.clone(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["missing".to_string()]), + &LogFieldSelector::LogAttribute(vec!["target".to_string()]), + ); + } + assert_eq!(log, snapshot_before); + } + + #[test] + fn move_field_unspecified_endpoints_noop() { + let mut log = make_log(); + let snapshot_before = log.clone(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::Simple(LogField::Unspecified), + &LogFieldSelector::LogAttribute(vec!["x".to_string()]), + ); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), + &LogFieldSelector::Simple(LogField::Unspecified), + ); + } + assert_eq!(log, snapshot_before); + } + + // --- Custom mappings ------------------------------------------------- + + #[test] + fn custom_body_path_is_honored() { + use vector_lib::lookup::lookup_v2::ConfigValuePath; + let mut log = LogEvent::default(); + log.insert("log.body", "custom"); + let mapping = FieldMapping { + body: ConfigValuePath::try_from("log.body".to_string()).unwrap(), + ..FieldMapping::default() + }; + let adapter = VectorLogAdapter::new(&mut log, &mapping); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::Body)) + .as_deref(), + Some("custom"), + ); + } + + #[test] + fn custom_attribute_root_is_honored() { + use vector_lib::lookup::lookup_v2::ConfigValuePath; + let mut log = LogEvent::default(); + log.insert("data.user_id", "42"); + let mapping = FieldMapping { + log_attributes: ConfigValuePath::try_from("data".to_string()).unwrap(), + ..FieldMapping::default() + }; + let adapter = VectorLogAdapter::new(&mut log, &mapping); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .as_deref(), + Some("42"), + ); + } +} diff --git a/src/transforms/policy/config.rs b/src/transforms/policy/config.rs new file mode 100644 index 0000000000000..fb3a192dc26b8 --- /dev/null +++ b/src/transforms/policy/config.rs @@ -0,0 +1,160 @@ +//! Configuration for the `policy` transform. + +use std::path::PathBuf; +use std::sync::Arc; + +use policy_rs::{FileProvider, PolicyEngine, PolicyRegistry}; +use vector_lib::{config::clone_input_definitions, configurable::configurable_component}; + +use crate::{ + config::{ + DataType, GenerateConfig, Input, OutputId, TransformConfig, TransformContext, + TransformOutput, + }, + schema, + transforms::Transform, +}; + +use super::field_mapping::FieldMapping; +use super::transform::Policy; + +/// Configuration for the `policy` transform. +/// +/// The transform delegates filtering, sampling, rate-limiting, and field +/// transformation to the external [`policy-rs`](https://github.com/usetero/policy-rs) +/// engine. Policies are loaded from a JSON file that the file-watcher +/// reloads automatically on changes. +#[configurable_component(transform( + "policy", + "Evaluate log events against a policy file and apply the resulting keep/drop/sample/rate-limit and field-transform actions." +))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct PolicyConfig { + /// Path to a JSON file containing the Tero Policy Specification document + /// to evaluate against. The file is watched and re-loaded on changes. + #[configurable(metadata(docs::examples = "/etc/vector/policies.json"))] + pub policies_path: PathBuf, + + /// Mapping between `policy-rs` log field selectors and paths within a + /// Vector `LogEvent`. + /// + /// Defaults follow OpenTelemetry semantic conventions so logs produced + /// by the `opentelemetry` source are matched without further + /// configuration. + #[configurable(derived)] + #[serde(default)] + pub field_mapping: FieldMapping, +} + +impl GenerateConfig for PolicyConfig { + fn generate_config() -> toml::Value { + toml::Value::try_from(Self { + policies_path: PathBuf::from("/etc/vector/policies.json"), + field_mapping: FieldMapping::default(), + }) + .unwrap() + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "policy")] +impl TransformConfig for PolicyConfig { + async fn build(&self, _context: &TransformContext) -> crate::Result { + let registry = PolicyRegistry::new(); + let provider = FileProvider::new(&self.policies_path); + registry.subscribe(&provider).map_err(|error| { + format!( + "failed to load policies from {:?}: {error}", + self.policies_path + ) + })?; + + let policy = Policy::new( + Arc::new(registry), + Arc::new(PolicyEngine::new()), + Arc::new(self.field_mapping.clone()), + ); + Ok(Transform::event_task(policy)) + } + + fn input(&self) -> Input { + // Accept everything; metrics and traces pass through unchanged. + Input::all() + } + + fn outputs( + &self, + _: &TransformContext, + input_definitions: &[(OutputId, schema::Definition)], + ) -> Vec { + vec![TransformOutput::new( + DataType::all_bits(), + clone_input_definitions(input_definitions), + )] + } + + fn enable_concurrency(&self) -> bool { + // Rate-limit state lives inside a single `PolicyEngine` instance, so + // running multiple copies of the transform would split the + // per-window counters and silently raise the effective rate limit. + // Keep concurrency disabled until policy-rs offers a shared + // rate-limiter handle. + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generate_config() { + crate::test_util::test_generate_config::(); + } + + #[test] + fn deserialize_minimal() { + let config: PolicyConfig = toml::from_str( + r#" +policies_path = "/tmp/policies.json" +"#, + ) + .unwrap(); + assert_eq!(config.policies_path, PathBuf::from("/tmp/policies.json")); + assert_eq!(config.field_mapping, FieldMapping::default()); + } + + #[test] + fn deserialize_with_field_mapping_overrides() { + let config: PolicyConfig = toml::from_str( + r#" +policies_path = "/tmp/policies.json" +[field_mapping] +body = "log.body" +"#, + ) + .unwrap(); + assert_eq!(String::from(config.field_mapping.body.clone()), "log.body"); + } + + #[test] + fn deserialize_rejects_unknown_field() { + let result: Result = toml::from_str( + r#" +policies_path = "/tmp/policies.json" +unknown = "value" +"#, + ); + assert!( + result.is_err(), + "unknown top-level fields should be rejected" + ); + } + + #[test] + fn deserialize_requires_policies_path() { + let result: Result = toml::from_str(""); + assert!(result.is_err(), "policies_path is required"); + } +} diff --git a/src/transforms/policy/field_mapping.rs b/src/transforms/policy/field_mapping.rs new file mode 100644 index 0000000000000..c504c437d7015 --- /dev/null +++ b/src/transforms/policy/field_mapping.rs @@ -0,0 +1,182 @@ +//! Configurable mapping between `policy-rs` log field selectors and Vector +//! `LogEvent` paths. +//! +//! Vector's `LogEvent` is schema-less: the `policy` transform must therefore +//! be told where in each event the body, severity, attributes, etc. live. +//! Defaults follow OpenTelemetry semantic conventions so that events emitted +//! by Vector's `opentelemetry` source work out of the box. + +use vector_lib::{ + configurable::configurable_component, + lookup::lookup_v2::{ConfigValuePath, OwnedValuePath}, +}; + +/// Maps `policy-rs` log field selectors onto paths inside a Vector `LogEvent`. +/// +/// All defaults follow OpenTelemetry semantic conventions so that logs emitted +/// by the `opentelemetry` source are matched without additional configuration. +#[configurable_component] +#[derive(Clone, Debug, PartialEq, Eq)] +#[serde(deny_unknown_fields, default)] +pub struct FieldMapping { + /// Path to the log body (`LogField::Body`). Default: `.message`. + pub body: ConfigValuePath, + + /// Path to the severity-text field (`LogField::SeverityText`). Default: `.severity_text`. + pub severity_text: ConfigValuePath, + + /// Path to the trace ID field (`LogField::TraceId`). Default: `.trace_id`. + pub trace_id: ConfigValuePath, + + /// Path to the span ID field (`LogField::SpanId`). Default: `.span_id`. + pub span_id: ConfigValuePath, + + /// Path to the event name field (`LogField::EventName`). Default: `.event_name`. + pub event_name: ConfigValuePath, + + /// Path to the resource schema URL (`LogField::ResourceSchemaUrl`). Default: `.resource.schema_url`. + pub resource_schema_url: ConfigValuePath, + + /// Path to the scope schema URL (`LogField::ScopeSchemaUrl`). Default: `.scope.schema_url`. + pub scope_schema_url: ConfigValuePath, + + /// Root path for log attributes (`LogFieldSelector::LogAttribute`). Default: `.attributes`. + /// + /// The selector's path segments are appended below this root to form the full event path. + pub log_attributes: ConfigValuePath, + + /// Root path for resource attributes (`LogFieldSelector::ResourceAttribute`). Default: `.resource.attributes`. + pub resource_attributes: ConfigValuePath, + + /// Root path for scope attributes (`LogFieldSelector::ScopeAttribute`). Default: `.scope.attributes`. + pub scope_attributes: ConfigValuePath, +} + +impl Default for FieldMapping { + fn default() -> Self { + Self { + body: parse("message"), + severity_text: parse("severity_text"), + trace_id: parse("trace_id"), + span_id: parse("span_id"), + event_name: parse("event_name"), + resource_schema_url: parse("resource.schema_url"), + scope_schema_url: parse("scope.schema_url"), + log_attributes: parse("attributes"), + resource_attributes: parse("resource.attributes"), + scope_attributes: parse("scope.attributes"), + } + } +} + +impl FieldMapping { + /// Compose `root` with `segments`, returning a fresh `OwnedValuePath`. + /// + /// Used to map e.g. `LogAttribute(["http", "method"])` to + /// `.attributes.http.method` under the default mapping. + pub(crate) fn append_segments(root: &ConfigValuePath, segments: &[String]) -> OwnedValuePath { + let mut path = root.0.clone(); + for segment in segments { + path = path.with_field_appended(segment); + } + path + } +} + +fn parse(path: &str) -> ConfigValuePath { + // SAFETY: the inputs here are static, fully-controlled strings. A panic + // would be a developer error caught by the unit tests below, never a + // runtime input. + ConfigValuePath::try_from(path.to_string()) + .unwrap_or_else(|err| panic!("invalid default path {path:?}: {err}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_paths_render_as_expected() { + let m = FieldMapping::default(); + assert_eq!(String::from(m.body.clone()), "message"); + assert_eq!(String::from(m.severity_text.clone()), "severity_text"); + assert_eq!(String::from(m.trace_id.clone()), "trace_id"); + assert_eq!(String::from(m.span_id.clone()), "span_id"); + assert_eq!(String::from(m.event_name.clone()), "event_name"); + assert_eq!( + String::from(m.resource_schema_url.clone()), + "resource.schema_url" + ); + assert_eq!(String::from(m.scope_schema_url.clone()), "scope.schema_url"); + assert_eq!(String::from(m.log_attributes.clone()), "attributes"); + assert_eq!( + String::from(m.resource_attributes.clone()), + "resource.attributes" + ); + assert_eq!(String::from(m.scope_attributes.clone()), "scope.attributes"); + } + + #[test] + fn append_single_segment() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments(&m.log_attributes, &["user_id".to_string()]); + assert_eq!(path.to_string(), "attributes.user_id"); + } + + #[test] + fn append_nested_segments() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments( + &m.log_attributes, + &["http".to_string(), "method".to_string()], + ); + assert_eq!(path.to_string(), "attributes.http.method"); + } + + #[test] + fn append_no_segments_returns_root() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments(&m.resource_attributes, &[]); + assert_eq!(path.to_string(), "resource.attributes"); + } + + #[test] + fn segment_with_special_chars_is_quoted() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments(&m.log_attributes, &["http.method".to_string()]); + // The dot inside the segment must be escaped/quoted in the rendered path. + assert_eq!(path.to_string(), "attributes.\"http.method\""); + } + + #[test] + fn default_round_trips_through_serde() { + let original = FieldMapping::default(); + let json = serde_json::to_string(&original).expect("serialize"); + let parsed: FieldMapping = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(parsed, original); + } + + #[test] + fn empty_object_uses_defaults() { + let parsed: FieldMapping = serde_json::from_str("{}").expect("deserialize {}"); + assert_eq!(parsed, FieldMapping::default()); + } + + #[test] + fn partial_override_keeps_other_defaults() { + let parsed: FieldMapping = + serde_json::from_str(r#"{"body":"log.body"}"#).expect("deserialize"); + assert_eq!(String::from(parsed.body), "log.body"); + // Other fields fall back to defaults. + assert_eq!( + String::from(parsed.severity_text), + String::from(FieldMapping::default().severity_text) + ); + } + + #[test] + fn unknown_field_is_rejected() { + let result: Result = serde_json::from_str(r#"{"unknown":"x"}"#); + assert!(result.is_err(), "unknown fields should be rejected"); + } +} diff --git a/src/transforms/policy/internal_events.rs b/src/transforms/policy/internal_events.rs new file mode 100644 index 0000000000000..3fda3416bc5ac --- /dev/null +++ b/src/transforms/policy/internal_events.rs @@ -0,0 +1,53 @@ +//! Internal events emitted by the `policy` transform. +//! +//! These live inside the transform module so the file under +//! `src/internal_events/` stays untouched — any future schema change can be +//! confined to this one file. + +use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL}; + +/// Reason tag attached to `component_discarded_events_total` for events the +/// policy engine declined to forward. +#[derive(Debug, Clone, Copy)] +pub(crate) enum DropReason { + /// A matching policy's `keep` action is `"none"` — drop unconditionally. + PolicyDrop, + /// A percentage-sample policy decided not to keep this event. + SampleRejected, + /// A rate-limited policy's window quota is exhausted. + RateLimited, +} + +impl DropReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + DropReason::PolicyDrop => "policy_drop", + DropReason::SampleRejected => "policy_sample_rejected", + DropReason::RateLimited => "policy_rate_limited", + } + } +} + +pub(crate) fn emit_dropped(reason: DropReason) { + emit!(ComponentEventsDropped:: { + count: 1, + reason: reason.as_str(), + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reason_strings_are_stable() { + // These tag values flow into emitted metrics. Changing them would be + // a user-visible break, so the test exists to draw attention to it. + assert_eq!(DropReason::PolicyDrop.as_str(), "policy_drop"); + assert_eq!( + DropReason::SampleRejected.as_str(), + "policy_sample_rejected" + ); + assert_eq!(DropReason::RateLimited.as_str(), "policy_rate_limited"); + } +} diff --git a/src/transforms/policy/mod.rs b/src/transforms/policy/mod.rs new file mode 100644 index 0000000000000..e6267bda7006b --- /dev/null +++ b/src/transforms/policy/mod.rs @@ -0,0 +1,17 @@ +//! Transform that delegates filtering, sampling, rate-limiting, and field +//! transformation to the [`policy-rs`](https://github.com/usetero/policy-rs) +//! library. +//! +//! The transform is fully self-contained inside this directory — no other +//! file in the Vector tree depends on `policy-rs` types — so future updates +//! to the library are isolated to this module. + +mod adapter; +mod field_mapping; +mod internal_events; + +pub mod config; +pub mod transform; + +#[cfg(test)] +mod tests; diff --git a/src/transforms/policy/tests.rs b/src/transforms/policy/tests.rs new file mode 100644 index 0000000000000..097274cad16ff --- /dev/null +++ b/src/transforms/policy/tests.rs @@ -0,0 +1,677 @@ +//! End-to-end tests for the `policy` transform driven through Vector's +//! topology layer. +//! +//! These exercise the integration with the `policy-rs` engine using real +//! JSON policy files on disk and the same `create_topology` helper used by +//! every other transform's integration tests. + +use std::io::Write; +use std::path::Path; +use std::time::Duration; + +use tempfile::NamedTempFile; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use vector_lib::event::{LogEvent, Metric, MetricKind, MetricValue}; + +use crate::event::{Event, Value}; +use crate::test_util::components::init_test; +use crate::transforms::test::create_topology; + +use super::config::PolicyConfig; +use super::field_mapping::FieldMapping; + +/// Write `body` to a fresh NamedTempFile with the `.json` suffix and return +/// it. The caller must keep the handle alive for the test's duration — +/// dropping it deletes the file out from under the `FileProvider`. +fn write_policies(body: &str) -> NamedTempFile { + let mut file = tempfile::Builder::new() + .prefix("vector-policy-test-") + .suffix(".json") + .tempfile() + .expect("create temp file"); + file.write_all(body.as_bytes()).expect("write policies"); + file.flush().expect("flush policies"); + file +} + +fn policy_config(path: &Path) -> PolicyConfig { + PolicyConfig { + policies_path: path.to_path_buf(), + field_mapping: FieldMapping::default(), + } +} + +/// Initialize metrics + tracing and build a topology in one step. Every test +/// in this file needs both, so funnel the boilerplate through a single helper. +async fn build_topology( + config: PolicyConfig, +) -> ( + mpsc::Sender, + mpsc::Receiver, + crate::topology::RunningTopology, +) { + init_test(); + let (tx, rx) = mpsc::channel(8); + let (topology, out) = create_topology(ReceiverStream::new(rx), config).await; + (tx, out, topology) +} + +/// Build a representative log event with the fields the default +/// `FieldMapping` expects. +fn log(body: &str) -> Event { + let mut log = LogEvent::default(); + log.insert("message", body); + log.insert("severity_text", "INFO"); + log.into() +} + +/// Receive the next event with a generous timeout — failures here typically +/// mean the transform silently dropped the event rather than yielding it. +async fn recv(out: &mut mpsc::Receiver) -> Option { + tokio::time::timeout(Duration::from_secs(2), out.recv()) + .await + .ok() + .flatten() +} + +/// Confirm the next operation drops, with a timeout so a buggy passthrough +/// can't mask a failure. +async fn assert_no_event(out: &mut mpsc::Receiver) { + let result = tokio::time::timeout(Duration::from_millis(300), out.recv()).await; + assert!( + result.is_err(), + "expected no event but received {:?}", + result.ok().flatten() + ); +} + +const POLICY_DROP_DEBUG: &str = r#"{ + "policies": [ + { + "id": "drop-debug", + "name": "drop-debug", + "log": { + "match": [ + { "log_field": "body", "regex": "debug" } + ], + "keep": "none" + } + } + ] +}"#; + +const POLICY_KEEP_ALL_MATCHING: &str = r#"{ + "policies": [ + { + "id": "keep-errors", + "name": "keep-errors", + "log": { + "match": [ + { "log_field": "body", "regex": "error" } + ], + "keep": "all" + } + } + ] +}"#; + +const POLICY_SAMPLE_ZERO: &str = r#"{ + "policies": [ + { + "id": "drop-sampled", + "name": "drop-sampled", + "log": { + "match": [ + { "log_field": "body", "regex": "drop-me" } + ], + "keep": "0%" + } + } + ] +}"#; + +const POLICY_SAMPLE_HUNDRED: &str = r#"{ + "policies": [ + { + "id": "keep-sampled", + "name": "keep-sampled", + "log": { + "match": [ + { "log_field": "body", "regex": "keep-me" } + ], + "keep": "100%" + } + } + ] +}"#; + +const POLICY_RATE_LIMIT_ONE_PER_SEC: &str = r#"{ + "policies": [ + { + "id": "rate-limit-noisy", + "name": "rate-limit-noisy", + "log": { + "match": [ + { "log_field": "body", "regex": "noisy" } + ], + "keep": "1/s" + } + } + ] +}"#; + +const POLICY_REDACT_PASSWORD: &str = r#"{ + "policies": [ + { + "id": "redact-password", + "name": "redact-password", + "log": { + "match": [ + { "log_field": "body", "regex": "login" } + ], + "keep": "all", + "transform": { + "redact": [ + { "log_attribute": "password", "replacement": "[REDACTED]" } + ] + } + } + } + ] +}"#; + +const POLICY_REMOVE_DEBUG_TRACE: &str = r#"{ + "policies": [ + { + "id": "remove-debug-trace", + "name": "remove-debug-trace", + "log": { + "match": [ + { "log_field": "body", "regex": ".+" } + ], + "keep": "all", + "transform": { + "remove": [ + { "log_attribute": "debug_trace" } + ] + } + } + } + ] +}"#; + +const POLICY_RENAME_LEGACY: &str = r#"{ + "policies": [ + { + "id": "rename-legacy", + "name": "rename-legacy", + "log": { + "match": [ + { "log_field": "body", "regex": ".+" } + ], + "keep": "all", + "transform": { + "rename": [ + { "from_log_attribute": "usr", "to": "user_id", "upsert": true } + ] + } + } + } + ] +}"#; + +const POLICY_ADD_PROCESSED_BY: &str = r#"{ + "policies": [ + { + "id": "add-processed-by", + "name": "add-processed-by", + "log": { + "match": [ + { "log_field": "body", "regex": ".+" } + ], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": false } + ] + } + } + } + ] +}"#; + +// ============================================================================= +// Pass-through and basic filter behaviour. +// ============================================================================= + +#[tokio::test] +async fn empty_policy_file_passes_events_through() { + let policies = write_policies(r#"{ "policies": [] }"#); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("hello world")).await.unwrap(); + let received = recv(&mut out).await.expect("event passed through"); + let log_event = received.into_log(); + assert_eq!( + log_event.get("message").and_then(|v| v.as_str()), + Some("hello world".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn no_matching_policy_passes_event_through() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("regular informational event")).await.unwrap(); + let received = recv(&mut out) + .await + .expect("non-matching event passes through"); + let log_event = received.into_log(); + assert_eq!( + log_event.get("message").and_then(|v| v.as_str()), + Some("regular informational event".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn matched_drop_policy_discards_event() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("debug message")).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn matched_keep_policy_forwards_event() { + let policies = write_policies(POLICY_KEEP_ALL_MATCHING); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("an error occurred")).await.unwrap(); + let received = recv(&mut out).await.expect("matching keep-all forwards"); + assert_eq!( + received.into_log().get("message").and_then(|v| v.as_str()), + Some("an error occurred".into()) + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Sampling at the extremes (0% drops everything, 100% keeps everything) is +// deterministic and therefore safe to assert. Intermediate percentages are +// inherently probabilistic and intentionally not tested here. +// ============================================================================= + +#[tokio::test] +async fn sample_zero_percent_drops_all_matches() { + let policies = write_policies(POLICY_SAMPLE_ZERO); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + for _ in 0..5 { + tx.send(log("drop-me please")).await.unwrap(); + } + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn sample_hundred_percent_keeps_all_matches() { + let policies = write_policies(POLICY_SAMPLE_HUNDRED); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + for _ in 0..3 { + tx.send(log("keep-me please")).await.unwrap(); + } + for _ in 0..3 { + let received = recv(&mut out).await.expect("100% sample forwards"); + assert_eq!( + received.into_log().get("message").and_then(|v| v.as_str()), + Some("keep-me please".into()) + ); + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Rate-limit policy: the first matching event in the window passes; the +// rest are dropped. Asserting more precise quantities would race with the +// rate-limiter's per-window clock. +// ============================================================================= + +#[tokio::test] +async fn rate_limit_drops_after_first_event() { + let policies = write_policies(POLICY_RATE_LIMIT_ONE_PER_SEC); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + // Fire several matching events in quick succession. With 1/s, only the + // first should clear the limiter; subsequent ones are dropped. + for _ in 0..5 { + tx.send(log("noisy chatter")).await.unwrap(); + } + + let first = recv(&mut out).await.expect("first event passes"); + assert_eq!( + first.into_log().get("message").and_then(|v| v.as_str()), + Some("noisy chatter".into()) + ); + + // Nothing else should arrive within the 1-second window. + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Transform actions: redact, remove, rename, add. +// ============================================================================= + +#[tokio::test] +async fn redact_replaces_field_value() { + let policies = write_policies(POLICY_REDACT_PASSWORD); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "user login attempt"); + event.insert("severity_text", "INFO"); + event.insert("attributes.password", "super_secret"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("redact forwards transformed event"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.password").and_then(|v| v.as_str()), + Some("[REDACTED]".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn remove_deletes_field() { + let policies = write_policies(POLICY_REMOVE_DEBUG_TRACE); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.debug_trace", "stack trace here"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("remove forwards transformed event"); + let log = received.into_log(); + assert!(log.get("attributes.debug_trace").is_none()); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn rename_moves_field() { + let policies = write_policies(POLICY_RENAME_LEGACY); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.usr", "admin"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("rename forwards transformed event"); + let log = received.into_log(); + assert!(log.get("attributes.usr").is_none()); + assert_eq!( + log.get("attributes.user_id").and_then(|v| v.as_str()), + Some("admin".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn add_inserts_field() { + let policies = write_policies(POLICY_ADD_PROCESSED_BY); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("add forwards transformed event"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.processed_by").and_then(|v| v.as_str()), + Some("vector".into()) + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Non-log signals are forwarded untouched. +// ============================================================================= + +#[tokio::test] +async fn metric_event_passes_through_untouched() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let metric = Event::from(Metric::new( + "test_metric", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + )); + tx.send(metric.clone()).await.unwrap(); + + let received = recv(&mut out).await.expect("metric passes through"); + match received { + Event::Metric(_) => {} // expected + other => panic!("expected metric, got {other:?}"), + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Hot reload: rewriting the policies file changes the engine's behaviour +// without restarting the transform. +// ============================================================================= + +#[tokio::test] +async fn policies_reload_on_file_change() { + use std::io::Seek; + + // Start with a no-op policy file. + let mut policies = write_policies(r#"{ "policies": [] }"#); + let path = policies.path().to_path_buf(); + let config = policy_config(&path); + + let (tx, mut out, topology) = build_topology(config).await; + + // Initially: an event with body "noisy" passes (no policy matches). + tx.send(log("noisy event")).await.unwrap(); + let _ = recv(&mut out).await.expect("initial pass-through"); + + // Rewrite the file with a drop policy. The FileProvider watches the + // parent directory and reloads on change. + policies.as_file_mut().set_len(0).unwrap(); + policies.as_file_mut().rewind().unwrap(); + policies + .write_all( + r#"{ + "policies": [ + { + "id": "drop-noisy", + "name": "drop-noisy", + "log": { + "match": [{ "log_field": "body", "regex": "noisy" }], + "keep": "none" + } + } + ] + }"# + .as_bytes(), + ) + .unwrap(); + policies.as_file_mut().flush().unwrap(); + + // notify-driven reload is asynchronous: poll the transform up to a few + // seconds while the new policy installs. + let mut reloaded = false; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + tx.send(log("noisy event")).await.unwrap(); + match tokio::time::timeout(Duration::from_millis(250), out.recv()).await { + Err(_) => { + reloaded = true; + break; + } + Ok(Some(_)) => { + // Old policy still in effect; try again shortly. + tokio::time::sleep(Duration::from_millis(100)).await; + } + Ok(None) => break, + } + } + assert!( + reloaded, + "policy file change did not propagate within 5s; events still pass through" + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Robustness: events that don't carry the body field still flow through. +// ============================================================================= + +#[tokio::test] +async fn event_missing_body_field_passes_through() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + // No `message` field at all — the body matcher should miss and the + // event should pass through unchanged. + let mut event = LogEvent::default(); + event.insert("attributes.other", "value"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("body-less event passes through"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.other").and_then(|v| v.as_str()), + Some("value".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn non_string_attribute_satisfies_exists_matcher() { + // Body matchers operate on strings, but `exists` matchers must fire + // for non-string attribute values too — the engine asks + // `Matchable::field_exists`, which our adapter overrides to true for + // any present value type. + let policy = r#"{ + "policies": [ + { + "id": "drop-anything-with-count", + "name": "drop-anything-with-count", + "log": { + "match": [{ "log_attribute": "count", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + // Integer attribute — should still trigger the drop. + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.count", 42_i64); + tx.send(event.into()).await.unwrap(); + assert_no_event(&mut out).await; + + // Boolean attribute — should also trigger. + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.count", Value::Boolean(true)); + tx.send(event.into()).await.unwrap(); + assert_no_event(&mut out).await; + + // No `count` attribute — should pass through. + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + tx.send(event.into()).await.unwrap(); + let _ = recv(&mut out).await.expect("missing count -> pass through"); + + drop(tx); + topology.stop().await; +} diff --git a/src/transforms/policy/transform.rs b/src/transforms/policy/transform.rs new file mode 100644 index 0000000000000..1078dc13e3fa2 --- /dev/null +++ b/src/transforms/policy/transform.rs @@ -0,0 +1,105 @@ +//! The async task transform that drives the `policy-rs` engine. + +use std::pin::Pin; +use std::sync::Arc; + +use async_stream::stream; +use futures::{Stream, StreamExt}; +use policy_rs::{EvaluateResult, PolicyEngine, PolicyRegistry}; +use vector_lib::transform::TaskTransform; + +use crate::event::Event; + +use super::adapter::VectorLogAdapter; +use super::field_mapping::FieldMapping; +use super::internal_events::{DropReason, emit_dropped}; + +/// Per-task state for the `policy` transform. +/// +/// `PolicyRegistry` and `PolicyEngine` are wrapped in `Arc` so the +/// `TaskTransform` machinery can move the struct freely while preserving +/// rate-limit and stats state across reconfigurations. +#[derive(Clone)] +pub struct Policy { + registry: Arc, + engine: Arc, + mapping: Arc, +} + +impl Policy { + pub const fn new( + registry: Arc, + engine: Arc, + mapping: Arc, + ) -> Self { + Self { + registry, + engine, + mapping, + } + } +} + +impl TaskTransform for Policy { + fn transform( + self: Box, + mut input_rx: Pin + Send>>, + ) -> Pin + Send>> + where + Self: 'static, + { + let registry = Arc::clone(&self.registry); + let engine = Arc::clone(&self.engine); + let mapping = Arc::clone(&self.mapping); + + Box::pin(stream! { + while let Some(event) = input_rx.next().await { + match event { + Event::Log(mut log) => { + // Pull a fresh snapshot every event so live-reloaded + // policies take effect on the next pass. Snapshots + // are cheap (Arc clone of an immutable inner). + let snapshot = registry.snapshot(); + let result = { + let mut adapter = VectorLogAdapter::new(&mut log, &mapping); + engine.evaluate_and_transform(&snapshot, &mut adapter).await + }; + match result { + Ok(EvaluateResult::NoMatch) + | Ok(EvaluateResult::Keep { .. }) + | Ok(EvaluateResult::Sample { keep: true, .. }) + | Ok(EvaluateResult::RateLimit { allowed: true, .. }) => { + yield Event::Log(log); + } + Ok(EvaluateResult::Drop { .. }) => { + emit_dropped(DropReason::PolicyDrop); + } + Ok(EvaluateResult::Sample { keep: false, .. }) => { + emit_dropped(DropReason::SampleRejected); + } + Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { + emit_dropped(DropReason::RateLimited); + } + Err(error) => { + // Fail open: an evaluation error shouldn't + // silently drop telemetry. Log and pass the + // event through untouched. + error!( + message = "Policy evaluation failed; event passed through unchanged.", + %error, + ); + yield Event::Log(log); + } + } + } + other => { + // policy-rs only targets the log signal in this + // integration. Metrics and traces are forwarded + // untouched. + yield other; + } + } + } + }) + } +} From 1f262dd7e4e6706169e3bb6deb2e3fcb58e04e0f Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Tue, 26 May 2026 22:25:59 -0400 Subject: [PATCH 2/9] more tests --- .github/workflows/tero-release.yaml | 3 + .gitignore | 1 + Cargo.lock | 2 + Cargo.toml | 2 +- src/transforms/policy/adapter.rs | 232 +++++++++ src/transforms/policy/config.rs | 187 ++++++- src/transforms/policy/tests.rs | 746 +++++++++++++++++++++++++++- 7 files changed, 1143 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tero-release.yaml b/.github/workflows/tero-release.yaml index 4c687dfb26d22..8e26e7150a1e7 100644 --- a/.github/workflows/tero-release.yaml +++ b/.github/workflows/tero-release.yaml @@ -95,12 +95,15 @@ jobs: build-essential \ clang \ cmake \ + libboost-dev \ libclang-dev \ libpcre2-dev \ + libprotobuf-dev \ libsasl2-dev \ libssl-dev \ libxml2-dev \ pkg-config \ + protobuf-compiler \ ragel \ unzip # `target-*-linux-gnu` pulls in `vendored` which builds krb5/gssapi diff --git a/.gitignore b/.gitignore index a713eeb297f9a..24a8428749602 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ cargo-timing*.html **/*.rs.bk target +target-test tests/data/wasm/*/target bench_output.txt rust-analyzer.toml diff --git a/Cargo.lock b/Cargo.lock index 26ee2a657b706..547fef2781ae2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8483,9 +8483,11 @@ dependencies = [ "prost-build 0.13.5", "rand 0.8.5", "regex", + "reqwest 0.12.28", "serde", "serde_json", "tokio", + "tonic 0.12.3", "tonic-build 0.12.3", "vectorscan-rs-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 133f31772db0c..052e34131533d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -828,7 +828,7 @@ transforms-window = [] transforms-log_to_metric = [] transforms-lua = ["dep:mlua", "vector-lib/lua"] transforms-metric_to_log = [] -transforms-policy = ["dep:policy-rs"] +transforms-policy = ["dep:policy-rs", "policy-rs/http", "policy-rs/grpc"] transforms-reduce = ["transforms-impl-reduce"] transforms-remap = [] transforms-route = [] diff --git a/src/transforms/policy/adapter.rs b/src/transforms/policy/adapter.rs index 950a845b58ed1..94940217099de 100644 --- a/src/transforms/policy/adapter.rs +++ b/src/transforms/policy/adapter.rs @@ -633,4 +633,236 @@ mod tests { Some("42"), ); } + + // --- Simple-field exhaustive coverage -------------------------------- + // + // Every `LogField` variant goes through the same `simple_path` table. + // The cases below confirm get/set/delete behave consistently across + // every variant we expose (Body and SeverityText already had bespoke + // tests; these round out the rest). + + /// One get/delete/set cycle for a simple field. Helper rather than + /// per-variant duplication so future `LogField` additions just append a + /// caller below. + fn assert_simple_round_trip(field: LogField, event_path: &str) { + let m = mapping(); + + // get + let mut log = LogEvent::default(); + log.insert(event_path, "initial"); + { + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(field)) + .as_deref(), + Some("initial"), + "get failed for {field:?} at {event_path:?}", + ); + } + + // delete + let was_present = { + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::Simple(field)) + }; + assert!(was_present, "delete returned false for {field:?}"); + assert!( + log.get(event_path).is_none(), + "delete left {event_path:?} present for {field:?}", + ); + + // set + { + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field(&LogFieldSelector::Simple(field), "after-set"); + } + assert_eq!( + log.get(event_path).and_then(|v| v.as_str()), + Some("after-set".into()), + "set didn't insert at {event_path:?} for {field:?}", + ); + } + + #[test] + fn round_trip_simple_body() { + assert_simple_round_trip(LogField::Body, "message"); + } + + #[test] + fn round_trip_simple_severity_text() { + assert_simple_round_trip(LogField::SeverityText, "severity_text"); + } + + #[test] + fn round_trip_simple_trace_id() { + assert_simple_round_trip(LogField::TraceId, "trace_id"); + } + + #[test] + fn round_trip_simple_span_id() { + assert_simple_round_trip(LogField::SpanId, "span_id"); + } + + #[test] + fn round_trip_simple_event_name() { + assert_simple_round_trip(LogField::EventName, "event_name"); + } + + #[test] + fn round_trip_simple_resource_schema_url() { + assert_simple_round_trip(LogField::ResourceSchemaUrl, "resource.schema_url"); + } + + #[test] + fn round_trip_simple_scope_schema_url() { + assert_simple_round_trip(LogField::ScopeSchemaUrl, "scope.schema_url"); + } + + // --- Resource / scope attribute Transformable coverage --------------- + // + // `LogAttribute` is already covered in detail. These mirror the same + // operations for the resource and scope namespaces so the namespace + // dispatch in `path_for` is exercised end-to-end. + + #[test] + fn set_resource_attribute_flat() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ResourceAttribute(vec!["region".to_string()]), + "us-east-1", + ); + } + assert_eq!( + log.get("resource.attributes.region") + .and_then(|v| v.as_str()), + Some("us-east-1".into()), + ); + } + + #[test] + fn set_resource_attribute_nested() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ResourceAttribute(vec!["k8s".to_string(), "pod".to_string()]), + "vector-0", + ); + } + assert_eq!( + log.get("resource.attributes.k8s.pod") + .and_then(|v| v.as_str()), + Some("vector-0".into()), + ); + } + + #[test] + fn delete_resource_attribute() { + let mut log = LogEvent::default(); + log.insert("resource.attributes.region", "us-east-1"); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::ResourceAttribute(vec![ + "region".to_string(), + ])) + }; + assert!(was_present); + assert!(log.get("resource.attributes.region").is_none()); + } + + #[test] + fn move_resource_attribute_within_namespace() { + let mut log = LogEvent::default(); + log.insert("resource.attributes.zone", "ap-south-1a"); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::ResourceAttribute(vec!["zone".to_string()]), + &LogFieldSelector::ResourceAttribute(vec!["availability_zone".to_string()]), + ); + } + assert!(log.get("resource.attributes.zone").is_none()); + assert_eq!( + log.get("resource.attributes.availability_zone") + .and_then(|v| v.as_str()), + Some("ap-south-1a".into()), + ); + } + + #[test] + fn set_scope_attribute_flat() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ScopeAttribute(vec!["lib_version".to_string()]), + "1.2.3", + ); + } + assert_eq!( + log.get("scope.attributes.lib_version") + .and_then(|v| v.as_str()), + Some("1.2.3".into()), + ); + } + + #[test] + fn set_scope_attribute_nested() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ScopeAttribute(vec!["vendor".to_string(), "name".to_string()]), + "otel", + ); + } + assert_eq!( + log.get("scope.attributes.vendor.name") + .and_then(|v| v.as_str()), + Some("otel".into()), + ); + } + + #[test] + fn delete_scope_attribute() { + let mut log = LogEvent::default(); + log.insert("scope.attributes.lib_version", "1.2.3"); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::ScopeAttribute(vec![ + "lib_version".to_string(), + ])) + }; + assert!(was_present); + assert!(log.get("scope.attributes.lib_version").is_none()); + } + + #[test] + fn move_scope_attribute_within_namespace() { + let mut log = LogEvent::default(); + log.insert("scope.attributes.legacy", "x"); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::ScopeAttribute(vec!["legacy".to_string()]), + &LogFieldSelector::ScopeAttribute(vec!["renamed".to_string()]), + ); + } + assert!(log.get("scope.attributes.legacy").is_none()); + assert_eq!( + log.get("scope.attributes.renamed").and_then(|v| v.as_str()), + Some("x".into()), + ); + } } diff --git a/src/transforms/policy/config.rs b/src/transforms/policy/config.rs index fb3a192dc26b8..0edfb9b2dd5f0 100644 --- a/src/transforms/policy/config.rs +++ b/src/transforms/policy/config.rs @@ -1,10 +1,20 @@ //! Configuration for the `policy` transform. -use std::path::PathBuf; +use std::cell::RefCell; use std::sync::Arc; -use policy_rs::{FileProvider, PolicyEngine, PolicyRegistry}; -use vector_lib::{config::clone_input_definitions, configurable::configurable_component}; +use policy_rs::{ + PolicyEngine, PolicyRegistry, + config::{FileProviderConfig, ProviderConfig as PolicyRsProviderConfig, register_providers}, +}; +use serde::{Deserialize, Serialize}; +use vector_lib::{ + config::clone_input_definitions, + configurable::{ + Configurable, GenerateError, Metadata, ToValue, configurable_component, + schema::{SchemaGenerator, SchemaObject}, + }, +}; use crate::{ config::{ @@ -22,8 +32,8 @@ use super::transform::Policy; /// /// The transform delegates filtering, sampling, rate-limiting, and field /// transformation to the external [`policy-rs`](https://github.com/usetero/policy-rs) -/// engine. Policies are loaded from a JSON file that the file-watcher -/// reloads automatically on changes. +/// engine. Policies are loaded from configured policy providers and +/// reloaded automatically when the providers report changes. #[configurable_component(transform( "policy", "Evaluate log events against a policy file and apply the resulting keep/drop/sample/rate-limit and field-transform actions." @@ -31,10 +41,12 @@ use super::transform::Policy; #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct PolicyConfig { - /// Path to a JSON file containing the Tero Policy Specification document - /// to evaluate against. The file is watched and re-loaded on changes. - #[configurable(metadata(docs::examples = "/etc/vector/policies.json"))] - pub policies_path: PathBuf, + /// Policy providers to register with the policy registry. + /// + /// Each provider uses policy-rs' tagged provider configuration format, + /// such as `{ type = "file", id = "local", path = "/etc/vector/policies.json" }`. + #[configurable(derived)] + pub policy_providers: Vec, /// Mapping between `policy-rs` log field selectors and paths within a /// Vector `LogEvent`. @@ -47,28 +59,84 @@ pub struct PolicyConfig { pub field_mapping: FieldMapping, } +/// Policy provider configuration. +/// +/// This wraps policy-rs' provider configuration so Vector can embed the +/// library's tagged provider enum directly in the transform config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(transparent)] +pub struct PolicyProviderConfig(PolicyRsProviderConfig); + +impl PolicyProviderConfig { + pub(super) fn file(id: impl Into, path: impl Into) -> Self { + Self(PolicyRsProviderConfig::File(FileProviderConfig { + id: id.into(), + path: path.into(), + })) + } + + fn into_inner(self) -> PolicyRsProviderConfig { + self.0 + } +} + +impl Configurable for PolicyProviderConfig { + fn metadata() -> Metadata { + let mut metadata = Metadata::default(); + metadata.set_description( + "Policy provider configuration using policy-rs' tagged provider format.", + ); + metadata + } + + fn generate_schema(_: &RefCell) -> Result { + Ok(SchemaObject::default()) + } +} + +impl ToValue for PolicyProviderConfig { + fn to_value(&self) -> serde_json::Value { + serde_json::to_value(self).expect("Could not convert policy provider config to JSON") + } +} + impl GenerateConfig for PolicyConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - policies_path: PathBuf::from("/etc/vector/policies.json"), + policy_providers: vec![PolicyProviderConfig::file( + "local", + "/etc/vector/policies.json", + )], field_mapping: FieldMapping::default(), }) .unwrap() } } +impl PolicyConfig { + fn provider_configs(&self) -> crate::Result> { + if !self.policy_providers.is_empty() { + return Ok(self + .policy_providers + .iter() + .cloned() + .map(PolicyProviderConfig::into_inner) + .collect()); + } + + Err("policy transform requires at least one policy provider".into()) + } +} + #[async_trait::async_trait] #[typetag::serde(name = "policy")] impl TransformConfig for PolicyConfig { async fn build(&self, _context: &TransformContext) -> crate::Result { let registry = PolicyRegistry::new(); - let provider = FileProvider::new(&self.policies_path); - registry.subscribe(&provider).map_err(|error| { - format!( - "failed to load policies from {:?}: {error}", - self.policies_path - ) - })?; + let provider_configs = self.provider_configs()?; + register_providers(&provider_configs, ®istry) + .await + .map_err(|error| format!("failed to register policy providers: {error}"))?; let policy = Policy::new( Arc::new(registry), @@ -117,11 +185,16 @@ mod tests { fn deserialize_minimal() { let config: PolicyConfig = toml::from_str( r#" -policies_path = "/tmp/policies.json" +[[policy_providers]] +id = "local" +type = "file" +path = "/tmp/policies.json" "#, ) .unwrap(); - assert_eq!(config.policies_path, PathBuf::from("/tmp/policies.json")); + let providers = config.provider_configs().unwrap(); + assert_eq!(providers.len(), 1); + assert!(matches!(providers[0], PolicyRsProviderConfig::File(_))); assert_eq!(config.field_mapping, FieldMapping::default()); } @@ -129,7 +202,11 @@ policies_path = "/tmp/policies.json" fn deserialize_with_field_mapping_overrides() { let config: PolicyConfig = toml::from_str( r#" -policies_path = "/tmp/policies.json" +[[policy_providers]] +id = "local" +type = "file" +path = "/tmp/policies.json" + [field_mapping] body = "log.body" "#, @@ -140,10 +217,18 @@ body = "log.body" #[test] fn deserialize_rejects_unknown_field() { + // The unknown top-level key must come BEFORE the `[[policy_providers]]` + // header — otherwise TOML attaches it to the array element instead of + // the root table, which would slip past `deny_unknown_fields` on + // `PolicyConfig`. let result: Result = toml::from_str( r#" -policies_path = "/tmp/policies.json" unknown = "value" + +[[policy_providers]] +id = "local" +type = "file" +path = "/tmp/policies.json" "#, ); assert!( @@ -153,8 +238,64 @@ unknown = "value" } #[test] - fn deserialize_requires_policies_path() { + fn deserialize_requires_policy_providers() { let result: Result = toml::from_str(""); - assert!(result.is_err(), "policies_path is required"); + assert!(result.is_err(), "policy_providers is required"); + } + + #[test] + fn empty_config_has_no_provider_configs() { + let config: PolicyConfig = toml::from_str( + r#" +policy_providers = [] +"#, + ) + .unwrap(); + let result = config.provider_configs(); + assert!(result.is_err(), "policy_providers must not be empty"); + } + + #[test] + fn deserialize_multiple_policy_providers() { + let config: PolicyConfig = toml::from_str( + r#" +[[policy_providers]] +id = "primary" +type = "file" +path = "/etc/vector/primary.json" + +[[policy_providers]] +id = "secondary" +type = "file" +path = "/etc/vector/secondary.json" +"#, + ) + .unwrap(); + let providers = config.provider_configs().unwrap(); + assert_eq!(providers.len(), 2); + assert!( + providers + .iter() + .all(|p| matches!(p, PolicyRsProviderConfig::File(_))) + ); + } + + #[test] + fn deserialize_rejects_unknown_provider_type() { + // policy-rs' `ProviderConfig` is a tagged enum on the `type` key. + // An unknown discriminator must fail to deserialize so users get a + // clear error rather than silently dropping the provider. + let result: Result = toml::from_str( + r#" +[[policy_providers]] +id = "bogus" +type = "nonexistent-provider-type" +path = "/dev/null" +"#, + ); + assert!( + result.is_err(), + "unknown provider `type` value should be rejected", + ); } } diff --git a/src/transforms/policy/tests.rs b/src/transforms/policy/tests.rs index 097274cad16ff..2dddde088df1a 100644 --- a/src/transforms/policy/tests.rs +++ b/src/transforms/policy/tests.rs @@ -18,7 +18,7 @@ use crate::event::{Event, Value}; use crate::test_util::components::init_test; use crate::transforms::test::create_topology; -use super::config::PolicyConfig; +use super::config::{PolicyConfig, PolicyProviderConfig}; use super::field_mapping::FieldMapping; /// Write `body` to a fresh NamedTempFile with the `.json` suffix and return @@ -37,7 +37,10 @@ fn write_policies(body: &str) -> NamedTempFile { fn policy_config(path: &Path) -> PolicyConfig { PolicyConfig { - policies_path: path.to_path_buf(), + policy_providers: vec![PolicyProviderConfig::file( + "local", + path.to_string_lossy().into_owned(), + )], field_mapping: FieldMapping::default(), } } @@ -201,11 +204,11 @@ const POLICY_REMOVE_DEBUG_TRACE: &str = r#"{ ] }"#; -const POLICY_RENAME_LEGACY: &str = r#"{ +const POLICY_RENAME_USER: &str = r#"{ "policies": [ { - "id": "rename-legacy", - "name": "rename-legacy", + "id": "rename-user", + "name": "rename-user", "log": { "match": [ { "log_field": "body", "regex": ".+" } @@ -448,7 +451,7 @@ async fn remove_deletes_field() { #[tokio::test] async fn rename_moves_field() { - let policies = write_policies(POLICY_RENAME_LEGACY); + let policies = write_policies(POLICY_RENAME_USER); let config = policy_config(policies.path()); let (tx, mut out, topology) = build_topology(config).await; @@ -675,3 +678,734 @@ async fn non_string_attribute_satisfies_exists_matcher() { drop(tx); topology.stop().await; } + +// ============================================================================= +// Matcher-type coverage. Each test confirms that a single matcher variant +// (other than `regex`/`exists`, which already had their own tests above) +// drives a `keep: "none"` policy as expected. +// ============================================================================= + +/// Build a one-policy `keep: "none"` JSON document from a single matcher +/// expressed as a raw JSON snippet (e.g. `r#""exact": "foo""#`). Keeps the +/// matcher tests tiny while still exercising the full deserialize → compile +/// → evaluate chain. +fn drop_policy_with_matcher(matcher_json: &str) -> String { + format!( + r#"{{ + "policies": [ + {{ + "id": "drop", + "name": "drop", + "log": {{ + "match": [{{ "log_field": "body", {matcher_json} }}], + "keep": "none" + }} + }} + ] +}}"# + ) +} + +#[tokio::test] +async fn exact_match_drops_on_full_string_equal() { + let policies = write_policies(&drop_policy_with_matcher(r#""exact": "hello""#)); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("hello")).await.unwrap(); + assert_no_event(&mut out).await; + + // A non-exact match (substring) must NOT drop. + tx.send(log("hello world")).await.unwrap(); + let _ = recv(&mut out).await.expect("non-exact body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn starts_with_match_drops() { + let policies = write_policies(&drop_policy_with_matcher(r#""starts_with": "DEBUG:""#)); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("DEBUG: trace info")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("INFO: app started")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("non-prefix body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn ends_with_match_drops() { + let policies = write_policies(&drop_policy_with_matcher( + r#""ends_with": "[health-check]""#, + )); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("ping [health-check]")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("user request received")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("non-suffix body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn contains_match_drops() { + let policies = write_policies(&drop_policy_with_matcher(r#""contains": "secret""#)); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("user said the secret word")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("nothing sensitive here")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("non-substring body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn negate_inverts_match_outcome() { + // Drop everything EXCEPT events whose body equals "keep". + let policies = write_policies(&drop_policy_with_matcher( + r#""exact": "keep", "negate": true"#, + )); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("anything")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("keep")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("negated match keeps the matching body"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn case_insensitive_match_drops_regardless_of_case() { + let policies = write_policies(&drop_policy_with_matcher( + r#""regex": "error", "case_insensitive": true"#, + )); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + for body in ["ERROR boom", "Error boom", "error boom", "ErRoR boom"] { + tx.send(log(body)).await.unwrap(); + assert_no_event(&mut out).await; + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Match-field namespace coverage: each policy field selector resolves to a +// LogEvent path, and we want one drop-policy test per supported selector to +// catch any regression in the adapter's `path_for` dispatch. +// ============================================================================= + +#[tokio::test] +async fn severity_text_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-debug-severity", + "name": "drop-debug-severity", + "log": { + "match": [{ "log_field": "severity_text", "exact": "DEBUG" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "anything"); + e.insert("severity_text", "DEBUG"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("anything")).await.unwrap(); // severity_text = INFO + let _ = recv(&mut out).await.expect("non-DEBUG passes"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn trace_id_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-trace", + "name": "drop-by-trace", + "log": { + "match": [{ "log_field": "trace_id", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("trace_id", "abc123"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("x")).await.unwrap(); // no trace_id + let _ = recv(&mut out).await.expect("no trace_id passes"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn span_id_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-span", + "name": "drop-by-span", + "log": { + "match": [{ "log_field": "span_id", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("span_id", "def456"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn event_name_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-event-name", + "name": "drop-by-event-name", + "log": { + "match": [{ "log_field": "event_name", "exact": "noisy.event" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("event_name", "noisy.event"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn resource_schema_url_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-resource-schema", + "name": "drop-by-resource-schema", + "log": { + "match": [{ "log_field": "resource_schema_url", "contains": "v1.0" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert( + "resource.schema_url", + "https://opentelemetry.io/schemas/v1.0", + ); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn scope_schema_url_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-scope-schema", + "name": "drop-by-scope-schema", + "log": { + "match": [{ "log_field": "scope_schema_url", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("scope.schema_url", "https://opentelemetry.io/schemas/foo"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn resource_attribute_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-service", + "name": "drop-by-service", + "log": { + "match": [{ "resource_attribute": "service.name", "exact": "noisy-service" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // The OTel-style "service.name" attribute key contains a literal dot, + // so we insert through a typed path that quotes the segment. + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert(r#"resource.attributes."service.name""#, "noisy-service"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn scope_attribute_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-scope-attr", + "name": "drop-by-scope-attr", + "log": { + "match": [{ "scope_attribute": "library", "exact": "deprecated-tracer" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("scope.attributes.library", "deprecated-tracer"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Multi-policy and multi-matcher semantics. +// ============================================================================= + +#[tokio::test] +async fn multiple_match_clauses_are_anded() { + // Drops only when BOTH the body matches AND the severity matches. + let policy = r#"{ + "policies": [ + { + "id": "drop-error-from-frontend", + "name": "drop-error-from-frontend", + "log": { + "match": [ + { "log_field": "body", "regex": "error" }, + { "log_field": "severity_text", "exact": "ERROR" } + ], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // Both clauses match -> dropped. + let mut e = LogEvent::default(); + e.insert("message", "an error occurred"); + e.insert("severity_text", "ERROR"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + // Body matches but severity does not -> passes through. + let mut e = LogEvent::default(); + e.insert("message", "an error occurred"); + e.insert("severity_text", "INFO"); + tx.send(e.into()).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("AND fails on severity mismatch"); + + // Severity matches but body does not -> passes through. + let mut e = LogEvent::default(); + e.insert("message", "all good"); + e.insert("severity_text", "ERROR"); + tx.send(e.into()).await.unwrap(); + let _ = recv(&mut out).await.expect("AND fails on body mismatch"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn transforms_from_all_matching_policies_apply() { + // Two policies both match the same event. The keep action comes from + // the winning policy (one of them), but per the policy-rs engine, the + // transforms from BOTH matching policies are applied. + let policy = r#"{ + "policies": [ + { + "id": "a-redact-secret", + "name": "a-redact-secret", + "log": { + "match": [{ "log_field": "body", "regex": "audit" }], + "keep": "all", + "transform": { + "redact": [ + { "log_attribute": "secret", "replacement": "[REDACTED]" } + ] + } + } + }, + { + "id": "b-add-processed-by", + "name": "b-add-processed-by", + "log": { + "match": [{ "log_field": "body", "regex": "audit" }], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": false } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "audit event"); + e.insert("severity_text", "INFO"); + e.insert("attributes.secret", "hunter2"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.secret").and_then(|v| v.as_str()), + Some("[REDACTED]".into()), + "redact from policy A should have applied", + ); + assert_eq!( + log.get("attributes.processed_by").and_then(|v| v.as_str()), + Some("vector".into()), + "add from policy B should have applied", + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn winning_policy_determines_keep_action() { + // Two policies both match. One says "keep: all", the other "keep: none". + // The engine sorts policies by ID alphabetically and selects a winner + // for the keep decision; for this test we just assert the event ends up + // in *some* deterministic state. Because the engine's choice is stable, + // we accept either outcome as long as it's consistent across the batch. + let policy = r#"{ + "policies": [ + { + "id": "aaa-drop", + "name": "aaa-drop", + "log": { + "match": [{ "log_field": "body", "regex": "contested" }], + "keep": "none" + } + }, + { + "id": "zzz-keep", + "name": "zzz-keep", + "log": { + "match": [{ "log_field": "body", "regex": "contested" }], + "keep": "all" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // Send a batch — every event must take the same path. If the winner is + // unstable across events the batch would mix drops and passes, which is + // what this test rules out. + for _ in 0..5 { + tx.send(log("contested event")).await.unwrap(); + } + let first = tokio::time::timeout(Duration::from_millis(500), out.recv()).await; + let dropped_consistently = first.is_err(); + let kept_consistently = first.as_ref().map(|r| r.is_some()).unwrap_or(false); + + if kept_consistently { + // First event was kept; the remaining four must also be kept. + for _ in 0..4 { + let _ = recv(&mut out) + .await + .expect("remaining contested events kept"); + } + } else if dropped_consistently { + // First event was dropped; the remaining four must also be dropped. + assert_no_event(&mut out).await; + } else { + panic!("ambiguous outcome: first event returned None during run"); + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Transform-op variant coverage. +// ============================================================================= + +#[tokio::test] +async fn redact_with_regex_redacts_substring_only() { + let policy = r#"{ + "policies": [ + { + "id": "redact-cc", + "name": "redact-cc", + "log": { + "match": [{ "log_field": "body", "regex": "card" }], + "keep": "all", + "transform": { + "redact": [ + { + "log_attribute": "card_number", + "replacement": "X", + "regex": "[0-9]" + } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "card event"); + e.insert("severity_text", "INFO"); + e.insert("attributes.card_number", "4242 1111 2222 3333"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.card_number").and_then(|v| v.as_str()), + Some("XXXX XXXX XXXX XXXX".into()), + "regex redact replaces each digit individually", + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn rename_without_upsert_skips_when_target_exists() { + let policy = r#"{ + "policies": [ + { + "id": "rename-no-upsert", + "name": "rename-no-upsert", + "log": { + "match": [{ "log_field": "body", "regex": "." }], + "keep": "all", + "transform": { + "rename": [ + { "from_log_attribute": "usr", "to": "user_id", "upsert": false } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // Target `user_id` already exists; rename without upsert must leave + // both keys in place (the source is NOT moved). + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("attributes.usr", "admin"); + e.insert("attributes.user_id", "existing"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.usr").and_then(|v| v.as_str()), + Some("admin".into()), + "source key must be preserved when upsert is false and target exists", + ); + assert_eq!( + log.get("attributes.user_id").and_then(|v| v.as_str()), + Some("existing".into()), + "existing target must NOT be overwritten", + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn add_with_upsert_overwrites_existing_value() { + let policy = r#"{ + "policies": [ + { + "id": "add-upsert", + "name": "add-upsert", + "log": { + "match": [{ "log_field": "body", "regex": "." }], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": true } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("attributes.processed_by", "something-else"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + assert_eq!( + received + .into_log() + .get("attributes.processed_by") + .and_then(|v| v.as_str()), + Some("vector".into()), + "upsert=true overwrites the existing value", + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Misc: trace pass-through + rate-limit-per-minute parse + provider list. +// ============================================================================= + +#[tokio::test] +async fn trace_event_passes_through_untouched() { + use vector_lib::event::TraceEvent; + + let policies = write_policies(POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let trace = Event::from(TraceEvent::default()); + tx.send(trace).await.unwrap(); + + match recv(&mut out).await { + Some(Event::Trace(_)) => {} + other => panic!("expected trace passthrough, got {other:?}"), + } + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn rate_limit_per_minute_first_event_passes() { + // We can't realistically test the *reset* of a per-minute window in a + // unit test (it would have to wait 60s). This guards the parse path + // and confirms the first matching event is still forwarded. + let policy = r#"{ + "policies": [ + { + "id": "rate-limit-per-minute", + "name": "rate-limit-per-minute", + "log": { + "match": [{ "log_field": "body", "regex": "minutely" }], + "keep": "60/m" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("minutely event")).await.unwrap(); + let _ = recv(&mut out).await.expect("first /m event passes"); + + drop(tx); + topology.stop().await; +} From 5c50add5d00aee5fe73e4a8b7de990b0581c12e5 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Wed, 27 May 2026 15:53:02 -0400 Subject: [PATCH 3/9] support OTLP native mode --- .../policy_transform_otel_mode.feature.md | 3 + src/transforms/policy/config.rs | 42 +- src/transforms/policy/mod.rs | 1 + src/transforms/policy/otlp_adapter.rs | 816 ++++++++++++++++++ src/transforms/policy/tests.rs | 549 +++++++++++- src/transforms/policy/transform.rs | 91 +- 6 files changed, 1468 insertions(+), 34 deletions(-) create mode 100644 changelog.d/policy_transform_otel_mode.feature.md create mode 100644 src/transforms/policy/otlp_adapter.rs diff --git a/changelog.d/policy_transform_otel_mode.feature.md b/changelog.d/policy_transform_otel_mode.feature.md new file mode 100644 index 0000000000000..de02b6c8674a7 --- /dev/null +++ b/changelog.d/policy_transform_otel_mode.feature.md @@ -0,0 +1,3 @@ +Added `mode: otel` option to the `policy` transform. When enabled, the transform treats each Vector event as an OTLP envelope (`{ resourceLogs: [...] }`) produced by the `opentelemetry` source with `use_otlp_decoding.logs = true`, iterates every `logRecord` inside, and applies policies per-record. Empty `scopeLogs` and `resourceLogs` entries are pruned automatically; if every record is filtered out the entire event is dropped. The default `mode: flat` preserves the previous one-event-per-decision behaviour. + +authors: jaronoff97 diff --git a/src/transforms/policy/config.rs b/src/transforms/policy/config.rs index 0edfb9b2dd5f0..63f5afe056a78 100644 --- a/src/transforms/policy/config.rs +++ b/src/transforms/policy/config.rs @@ -48,8 +48,22 @@ pub struct PolicyConfig { #[configurable(derived)] pub policy_providers: Vec, + /// How Vector events are mapped to policy-rs records. + /// + /// `flat` (the default) treats every Vector event as a single log + /// record and uses the configurable `field_mapping`. + /// + /// `otel` treats every Vector event as an OTLP envelope and iterates + /// `resourceLogs[].scopeLogs[].logRecords[]` internally, applying + /// policies per-record and pruning empty children. `field_mapping` + /// is ignored in this mode — OTLP field locations are fixed by the + /// protocol. + #[configurable(derived)] + #[serde(default)] + pub mode: PolicyMode, + /// Mapping between `policy-rs` log field selectors and paths within a - /// Vector `LogEvent`. + /// Vector `LogEvent`. Only used when `mode = flat`. /// /// Defaults follow OpenTelemetry semantic conventions so logs produced /// by the `opentelemetry` source are matched without further @@ -59,6 +73,30 @@ pub struct PolicyConfig { pub field_mapping: FieldMapping, } +/// Iteration mode for the `policy` transform. +#[configurable_component] +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum PolicyMode { + /// One Vector event maps to one log record. Use this mode for sources + /// that emit flat Vector events (the default for most sources). + /// + /// Policy field selectors are resolved through `field_mapping`. + #[default] + Flat, + + /// One Vector event is an OTLP envelope (`{ resourceLogs: [...] }`). + /// + /// The transform iterates every `logRecord` inside the envelope, + /// applies policies per-record, and prunes empty `scopeLogs` and + /// `resourceLogs` entries. If every record is filtered out, the + /// entire envelope event is dropped. + /// + /// Use this mode with Vector's `opentelemetry` source when + /// `use_otlp_decoding.logs` is `true`. + Otel, +} + /// Policy provider configuration. /// /// This wraps policy-rs' provider configuration so Vector can embed the @@ -107,6 +145,7 @@ impl GenerateConfig for PolicyConfig { "local", "/etc/vector/policies.json", )], + mode: PolicyMode::default(), field_mapping: FieldMapping::default(), }) .unwrap() @@ -142,6 +181,7 @@ impl TransformConfig for PolicyConfig { Arc::new(registry), Arc::new(PolicyEngine::new()), Arc::new(self.field_mapping.clone()), + self.mode, ); Ok(Transform::event_task(policy)) } diff --git a/src/transforms/policy/mod.rs b/src/transforms/policy/mod.rs index e6267bda7006b..e86975f977601 100644 --- a/src/transforms/policy/mod.rs +++ b/src/transforms/policy/mod.rs @@ -9,6 +9,7 @@ mod adapter; mod field_mapping; mod internal_events; +mod otlp_adapter; pub mod config; pub mod transform; diff --git a/src/transforms/policy/otlp_adapter.rs b/src/transforms/policy/otlp_adapter.rs new file mode 100644 index 0000000000000..97fb065dc97c8 --- /dev/null +++ b/src/transforms/policy/otlp_adapter.rs @@ -0,0 +1,816 @@ +//! OTLP envelope adapter for the `policy` transform. +//! +//! When the transform runs in `mode: otel`, each incoming Vector event is +//! treated as an OTLP envelope (`{ resourceLogs: [...] }`) emitted by +//! Vector's `opentelemetry` source with `use_otlp_decoding.logs = true`. +//! This module iterates every `logRecord` inside the envelope, evaluates +//! it through `policy-rs` via [`OtlpLogAdapter`], filters in place, and +//! prunes empty `scopeLogs` / `resourceLogs` entries. +//! +//! Differences from [`super::adapter::VectorLogAdapter`]: +//! +//! * Field names are camelCase per the proto3 JSON mapping +//! (`severityText`, not `severity_text`). +//! * `body` is an OTLP `AnyValue`, so `body.stringValue` (and other +//! variants) is the real path to the value. +//! * Attributes are always arrays of `{ key, value: AnyValue }` — not +//! key-value maps. Lookup is a linear scan by key. +//! * Resource and scope are read-only in this mode: mutating cloned +//! siblings would not affect the underlying envelope, so +//! `set_field` / `delete_field` / `move_field` on +//! `ResourceAttribute` / `ScopeAttribute` selectors are no-ops. We +//! can lift this restriction later by restructuring the iteration if +//! real users need it. +//! * Multi-segment `LogAttribute(["http", "method"])` paths are +//! dot-joined and looked up as the single OTel key `"http.method"`. +//! Walking `kvlistValue` for true nesting is a future extension. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::LogField; +use policy_rs::{ + EvaluateResult, LogFieldSelector, Matchable, PolicyEngine, PolicySnapshot, Transformable, + engine::LogSignal, +}; +use vector_lib::event::{LogEvent, ObjectMap, Value}; + +use super::internal_events::{DropReason, emit_dropped}; + +/// Iterate every record inside an OTLP envelope event, applying policies +/// per-record. Returns `Some(log)` to forward (with mutated and possibly +/// pruned contents), or `None` if every record was filtered out and the +/// envelope should be dropped entirely. +/// +/// If the event is not envelope-shaped (no `resourceLogs` key, or it's +/// not an array), the event is forwarded unchanged so users who +/// accidentally send non-envelope events through an `otel`-mode transform +/// don't lose data silently. +pub(super) async fn evaluate_envelope( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + mut log: LogEvent, +) -> Option { + let Some(resource_logs) = log.get_mut("resourceLogs").and_then(Value::as_array_mut) else { + // Not an envelope. Pass through. + return Some(log); + }; + + let mut i = 0; + while i < resource_logs.len() { + // Clone the resource sub-object and the resourceLogs entry's + // schemaUrl so the adapter can hold immutable refs to them without + // aliasing the mutable borrow we'll take of `scopeLogs` below. + let resource = resource_logs[i].get("resource").cloned(); + let resource_schema_url = resource_logs[i].get("schemaUrl").cloned(); + + let mut prune_this_rl = false; + + if let Some(scope_logs) = resource_logs[i] + .get_mut("scopeLogs") + .and_then(Value::as_array_mut) + { + let mut j = 0; + while j < scope_logs.len() { + let scope = scope_logs[j].get("scope").cloned(); + let scope_schema_url = scope_logs[j].get("schemaUrl").cloned(); + + let mut prune_this_sl = false; + + if let Some(records) = scope_logs[j] + .get_mut("logRecords") + .and_then(Value::as_array_mut) + { + let mut k = 0; + while k < records.len() { + let result = { + let mut adapter = OtlpLogAdapter::new( + &mut records[k], + resource.as_ref(), + scope.as_ref(), + resource_schema_url.as_ref(), + scope_schema_url.as_ref(), + ); + engine.evaluate_and_transform(snapshot, &mut adapter).await + }; + + match result { + Ok(EvaluateResult::NoMatch) + | Ok(EvaluateResult::Keep { .. }) + | Ok(EvaluateResult::Sample { keep: true, .. }) + | Ok(EvaluateResult::RateLimit { allowed: true, .. }) => { + k += 1; + } + Ok(EvaluateResult::Drop { .. }) => { + records.remove(k); + emit_dropped(DropReason::PolicyDrop); + } + Ok(EvaluateResult::Sample { keep: false, .. }) => { + records.remove(k); + emit_dropped(DropReason::SampleRejected); + } + Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { + records.remove(k); + emit_dropped(DropReason::RateLimited); + } + Err(error) => { + error!( + message = "Policy evaluation failed; OTLP record passed through unchanged.", + %error, + ); + k += 1; + } + } + } + prune_this_sl = records.is_empty(); + } + + if prune_this_sl { + scope_logs.remove(j); + } else { + j += 1; + } + } + prune_this_rl = scope_logs.is_empty(); + } + + if prune_this_rl { + resource_logs.remove(i); + } else { + i += 1; + } + } + + if resource_logs.is_empty() { + None + } else { + Some(log) + } +} + +/// Adapter exposing a single OTLP `logRecord` (plus its parent resource +/// and scope) to the `policy-rs` engine. +pub(super) struct OtlpLogAdapter<'a> { + log_record: &'a mut Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, +} + +impl<'a> OtlpLogAdapter<'a> { + pub(super) const fn new( + log_record: &'a mut Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, + ) -> Self { + Self { + log_record, + resource, + scope, + resource_schema_url, + scope_schema_url, + } + } + + /// Look up the value at a simple-field selector for reads. Returns the + /// `&Value` (still wrapped) so callers can choose between + /// stringification (for `get_field`) and presence-only checks (for + /// `field_exists`). + fn simple_value(&self, field: LogField) -> Option<&Value> { + match field { + LogField::Body => self.log_record.get("body"), + LogField::SeverityText => self.log_record.get("severityText"), + LogField::TraceId => self.log_record.get("traceId"), + LogField::SpanId => self.log_record.get("spanId"), + LogField::EventName => self.log_record.get("eventName"), + LogField::ResourceSchemaUrl => self.resource_schema_url, + LogField::ScopeSchemaUrl => self.scope_schema_url, + LogField::Unspecified => None, + } + } + + /// Look up the attributes array for an attribute-namespace selector. + fn attributes_for(&self, selector: &LogFieldSelector) -> Option<&Value> { + match selector { + LogFieldSelector::LogAttribute(_) => self.log_record.get("attributes"), + LogFieldSelector::ResourceAttribute(_) => { + self.resource.and_then(|r| r.get("attributes")) + } + LogFieldSelector::ScopeAttribute(_) => self.scope.and_then(|s| s.get("attributes")), + LogFieldSelector::Simple(_) => None, + } + } +} + +impl Matchable for OtlpLogAdapter<'_> { + type Signal = LogSignal; + + fn get_field(&self, field: &LogFieldSelector) -> Option> { + match field { + LogFieldSelector::Simple(LogField::Body) => { + self.log_record.get("body").and_then(any_value_to_string) + } + LogFieldSelector::Simple(simple) => { + self.simple_value(*simple).and_then(plain_value_to_string) + } + LogFieldSelector::LogAttribute(path) + | LogFieldSelector::ResourceAttribute(path) + | LogFieldSelector::ScopeAttribute(path) => { + let attrs = self.attributes_for(field)?; + find_attribute_value(attrs, &join_path(path)) + } + } + } + + fn field_exists(&self, field: &LogFieldSelector) -> bool { + match field { + LogFieldSelector::Simple(LogField::Unspecified) => false, + LogFieldSelector::Simple(simple) => self.simple_value(*simple).is_some(), + LogFieldSelector::LogAttribute(path) + | LogFieldSelector::ResourceAttribute(path) + | LogFieldSelector::ScopeAttribute(path) => match self.attributes_for(field) { + Some(attrs) => attribute_exists(attrs, &join_path(path)), + None => false, + }, + } + } +} + +impl Transformable for OtlpLogAdapter<'_> { + fn set_field(&mut self, field: &LogFieldSelector, value: &str) { + match field { + LogFieldSelector::Simple(LogField::Body) => { + self.log_record.insert("body", make_string_any_value(value)); + } + LogFieldSelector::Simple(LogField::SeverityText) => { + insert_simple_string(self.log_record, "severityText", value); + } + LogFieldSelector::Simple(LogField::TraceId) => { + insert_simple_string(self.log_record, "traceId", value); + } + LogFieldSelector::Simple(LogField::SpanId) => { + insert_simple_string(self.log_record, "spanId", value); + } + LogFieldSelector::Simple(LogField::EventName) => { + insert_simple_string(self.log_record, "eventName", value); + } + // Simple fields whose backing storage isn't on the log record + // itself (schema URLs live on the parent entries) are read-only + // for the same reason ResourceAttribute / ScopeAttribute are: + // we only hold immutable refs. + LogFieldSelector::Simple(_) => {} + LogFieldSelector::LogAttribute(path) => { + let key = join_path(path); + ensure_attributes_array(self.log_record); + if let Some(attrs) = self + .log_record + .get_mut("attributes") + .and_then(Value::as_array_mut) + { + set_attribute(attrs, &key, value); + } + } + LogFieldSelector::ResourceAttribute(_) | LogFieldSelector::ScopeAttribute(_) => { + // Read-only in OTel mode; see module docstring. + } + } + } + + fn delete_field(&mut self, field: &LogFieldSelector) -> bool { + match field { + LogFieldSelector::Simple(LogField::Body) => self.log_record_remove("body"), + LogFieldSelector::Simple(LogField::SeverityText) => { + self.log_record_remove("severityText") + } + LogFieldSelector::Simple(LogField::TraceId) => self.log_record_remove("traceId"), + LogFieldSelector::Simple(LogField::SpanId) => self.log_record_remove("spanId"), + LogFieldSelector::Simple(LogField::EventName) => self.log_record_remove("eventName"), + LogFieldSelector::Simple(_) => false, + LogFieldSelector::LogAttribute(path) => { + let key = join_path(path); + self.log_record + .get_mut("attributes") + .and_then(Value::as_array_mut) + .map(|attrs| remove_attribute(attrs, &key)) + .unwrap_or(false) + } + LogFieldSelector::ResourceAttribute(_) | LogFieldSelector::ScopeAttribute(_) => false, + } + } + + fn move_field(&mut self, from: &LogFieldSelector, to: &LogFieldSelector) { + // The engine guarantees `from` exists and `to` does not. In OTel + // mode we only support moves within the log-record's own + // attributes array. + let (LogFieldSelector::LogAttribute(from_path), LogFieldSelector::LogAttribute(to_path)) = + (from, to) + else { + return; + }; + let from_key = join_path(from_path); + let to_key = join_path(to_path); + if let Some(attrs) = self + .log_record + .get_mut("attributes") + .and_then(Value::as_array_mut) + && let Some(idx) = attribute_index(attrs, &from_key) + { + let entry = attrs.remove(idx); + if let Some(value) = entry.as_object().and_then(|o| o.get("value")).cloned() { + attrs.push(make_attribute_entry(&to_key, value)); + } + } + } +} + +impl OtlpLogAdapter<'_> { + fn log_record_remove(&mut self, key: &str) -> bool { + match self.log_record.as_object_mut() { + Some(obj) => obj.remove(key).is_some(), + None => false, + } + } +} + +// ============================================================================= +// Value-shape helpers. +// ============================================================================= + +/// Coerce an OTLP `AnyValue` object to a string for matching purposes. +/// +/// `AnyValue` is a oneof with seven variants (`stringValue`, `intValue`, +/// `boolValue`, `doubleValue`, `bytesValue`, `arrayValue`, `kvlistValue`). +/// Scalar variants get stringified; container/binary variants return +/// `None` because there's no canonical text form for matching. +fn any_value_to_string(value: &Value) -> Option> { + let obj = value.as_object()?; + if let Some(v) = obj.get("stringValue") + && let Some(s) = v.as_str() + { + return Some(s); + } + if let Some(v) = obj.get("intValue") + && let Some(i) = v.as_integer() + { + return Some(Cow::Owned(i.to_string())); + } + if let Some(v) = obj.get("boolValue") + && let Some(b) = v.as_boolean() + { + return Some(Cow::Borrowed(if b { "true" } else { "false" })); + } + if let Some(v) = obj.get("doubleValue") + && let Some(f) = v.as_float() + { + return Some(Cow::Owned(f.to_string())); + } + None +} + +/// Coerce a plain (non-`AnyValue`) `Value` to a string for matching — +/// used for simple OTLP fields like `severityText`, `traceId`, `spanId`, +/// which are flat strings rather than wrapped `AnyValue` objects. +fn plain_value_to_string(value: &Value) -> Option> { + match value { + Value::Bytes(_) => value.as_str(), + Value::Integer(i) => Some(Cow::Owned(i.to_string())), + Value::Float(f) => Some(Cow::Owned(f.to_string())), + Value::Boolean(b) => Some(Cow::Borrowed(if *b { "true" } else { "false" })), + _ => None, + } +} + +/// Linear scan of an OTLP attributes array for the entry whose key +/// matches. +fn find_attribute_value<'a>(attrs: &'a Value, key: &str) -> Option> { + let array = attrs.as_array()?; + for item in array { + if attribute_key(item) == Some(key) { + return item + .as_object() + .and_then(|o| o.get("value")) + .and_then(any_value_to_string); + } + } + None +} + +/// Like [`find_attribute_value`] but returns whether the key exists at +/// all — used by `field_exists` so OTLP attributes with non-string +/// `AnyValue` variants (`intValue`, `boolValue`, `arrayValue`, …) still +/// satisfy `exists: true` matchers. +fn attribute_exists(attrs: &Value, key: &str) -> bool { + attrs + .as_array() + .map(|array| array.iter().any(|item| attribute_key(item) == Some(key))) + .unwrap_or(false) +} + +fn attribute_index(attrs: &[Value], key: &str) -> Option { + attrs + .iter() + .position(|item| attribute_key(item) == Some(key)) +} + +fn attribute_key(item: &Value) -> Option<&str> { + item.as_object() + .and_then(|o| o.get("key")) + .and_then(|v| match v { + Value::Bytes(b) => std::str::from_utf8(b).ok(), + _ => None, + }) +} + +/// Build an `AnyValue` object wrapping a string. +fn make_string_any_value(value: &str) -> Value { + let mut obj = ObjectMap::new(); + obj.insert("stringValue".into(), Value::from(value.to_string())); + Value::Object(obj) +} + +/// Build a single OTLP attribute entry `{ key, value }`. +fn make_attribute_entry(key: &str, value: Value) -> Value { + let mut obj = ObjectMap::new(); + obj.insert("key".into(), Value::from(key.to_string())); + obj.insert("value".into(), value); + Value::Object(obj) +} + +/// Insert (or overwrite) an attribute entry by key, wrapping the value +/// as a string `AnyValue`. +fn set_attribute(attrs: &mut Vec, key: &str, value: &str) { + let any_value = make_string_any_value(value); + if let Some(idx) = attribute_index(attrs, key) { + if let Some(obj) = attrs[idx].as_object_mut() { + obj.insert("value".into(), any_value); + } + } else { + attrs.push(make_attribute_entry(key, any_value)); + } +} + +/// Remove an attribute entry by key. Returns whether an entry was +/// removed. +fn remove_attribute(attrs: &mut Vec, key: &str) -> bool { + match attribute_index(attrs, key) { + Some(idx) => { + attrs.remove(idx); + true + } + None => false, + } +} + +/// Ensure the log record has an `attributes` array — create an empty one +/// if absent so `set_attribute` has somewhere to push. +fn ensure_attributes_array(log_record: &mut Value) { + let needs_init = log_record + .as_object() + .and_then(|o| o.get("attributes")) + .map(|v| !matches!(v, Value::Array(_))) + .unwrap_or(true); + if needs_init && let Some(obj) = log_record.as_object_mut() { + obj.insert("attributes".into(), Value::Array(Vec::new())); + } +} + +/// `policy-rs` attribute selectors are multi-segment paths. In OTel mode +/// we collapse them with a literal dot — the OTel convention is that +/// nested attribute keys (e.g. `"service.name"`) live as a single +/// flat key with embedded dots. +fn join_path(segments: &[String]) -> String { + segments.join(".") +} + +fn insert_simple_string(record: &mut Value, key: &str, value: &str) { + if let Some(obj) = record.as_object_mut() { + obj.insert(key.into(), Value::from(value.to_string())); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a single OTLP log record `Value` with the given body and a + /// `user_id` string attribute. Keeps the test bodies tiny. + fn record_with_body_and_attr(body: &str, attr_key: &str, attr_val: &str) -> Value { + let mut record = ObjectMap::new(); + record.insert("body".into(), make_string_any_value(body)); + record.insert("severityText".into(), Value::from("INFO".to_string())); + record.insert( + "attributes".into(), + Value::Array(vec![make_attribute_entry( + attr_key, + make_string_any_value(attr_val), + )]), + ); + Value::Object(record) + } + + fn resource_with_attr(key: &str, val: &str) -> Value { + let mut resource = ObjectMap::new(); + resource.insert( + "attributes".into(), + Value::Array(vec![make_attribute_entry(key, make_string_any_value(val))]), + ); + Value::Object(resource) + } + + fn scope_with_attr(key: &str, val: &str) -> Value { + let mut scope = ObjectMap::new(); + scope.insert("name".into(), Value::from("test".to_string())); + scope.insert( + "attributes".into(), + Value::Array(vec![make_attribute_entry(key, make_string_any_value(val))]), + ); + Value::Object(scope) + } + + #[test] + fn any_value_string() { + let v = make_string_any_value("hello"); + assert_eq!(any_value_to_string(&v).as_deref(), Some("hello")); + } + + #[test] + fn any_value_int_stringifies() { + let mut obj = ObjectMap::new(); + obj.insert("intValue".into(), Value::Integer(42)); + assert_eq!( + any_value_to_string(&Value::Object(obj)).as_deref(), + Some("42"), + ); + } + + #[test] + fn any_value_bool_stringifies() { + let mut obj = ObjectMap::new(); + obj.insert("boolValue".into(), Value::Boolean(true)); + assert_eq!( + any_value_to_string(&Value::Object(obj)).as_deref(), + Some("true"), + ); + } + + #[test] + fn any_value_double_stringifies() { + let mut obj = ObjectMap::new(); + obj.insert( + "doubleValue".into(), + Value::Float(ordered_float::NotNan::new(1.5).unwrap()), + ); + assert!(matches!( + any_value_to_string(&Value::Object(obj)).as_deref(), + Some(s) if s.starts_with("1.5"), + )); + } + + #[test] + fn any_value_array_returns_none_for_matching() { + let mut obj = ObjectMap::new(); + obj.insert("arrayValue".into(), Value::Array(vec![])); + assert_eq!(any_value_to_string(&Value::Object(obj)), None); + } + + #[test] + fn get_body_string() { + let mut record = record_with_body_and_attr("hi", "user_id", "42"); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::Body)) + .as_deref(), + Some("hi"), + ); + } + + #[test] + fn get_log_attribute_flat() { + let mut record = record_with_body_and_attr("hi", "user_id", "42"); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .as_deref(), + Some("42"), + ); + } + + #[test] + fn get_log_attribute_nested_dot_joins() { + let mut record = record_with_body_and_attr("x", "http.method", "GET"); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec![ + "http".to_string(), + "method".to_string() + ])) + .as_deref(), + Some("GET"), + ); + } + + #[test] + fn get_resource_attribute_via_immutable_ref() { + let mut record = record_with_body_and_attr("x", "noise", "v"); + let resource = resource_with_attr("service.name", "frontend"); + let adapter = OtlpLogAdapter::new(&mut record, Some(&resource), None, None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::ResourceAttribute(vec![ + "service.name".to_string() + ])) + .as_deref(), + Some("frontend"), + ); + } + + #[test] + fn get_scope_attribute_via_immutable_ref() { + let mut record = record_with_body_and_attr("x", "noise", "v"); + let scope = scope_with_attr("library", "tracer"); + let adapter = OtlpLogAdapter::new(&mut record, None, Some(&scope), None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::ScopeAttribute(vec![ + "library".to_string() + ])) + .as_deref(), + Some("tracer"), + ); + } + + #[test] + fn get_resource_schema_url() { + let mut record = record_with_body_and_attr("x", "noise", "v"); + let schema = Value::from("https://opentelemetry.io/schemas/1.0".to_string()); + let adapter = OtlpLogAdapter::new(&mut record, None, None, Some(&schema), None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::ResourceSchemaUrl)) + .as_deref(), + Some("https://opentelemetry.io/schemas/1.0"), + ); + } + + #[test] + fn field_exists_for_non_string_attribute() { + // intValue attribute must satisfy exists matchers even though + // get_field returns Some("42"). + let mut record = ObjectMap::new(); + record.insert("body".into(), make_string_any_value("x")); + let mut attr_value = ObjectMap::new(); + attr_value.insert("intValue".into(), Value::Integer(42)); + record.insert( + "attributes".into(), + Value::Array(vec![make_attribute_entry( + "count", + Value::Object(attr_value), + )]), + ); + let mut record = Value::Object(record); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["count".to_string()]))); + } + + #[test] + fn set_attribute_overwrites_existing() { + let mut record = record_with_body_and_attr("x", "user_id", "42"); + { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.set_field( + &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), + "99", + ); + } + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .as_deref(), + Some("99"), + ); + } + + #[test] + fn set_attribute_creates_when_absent() { + let mut record = ObjectMap::new(); + record.insert("body".into(), make_string_any_value("x")); + let mut record = Value::Object(record); + { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.set_field( + &LogFieldSelector::LogAttribute(vec!["new".to_string()]), + "v", + ); + } + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["new".to_string()])) + .as_deref(), + Some("v"), + ); + } + + #[test] + fn set_simple_body_replaces_any_value() { + let mut record = record_with_body_and_attr("old", "k", "v"); + { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.set_field(&LogFieldSelector::Simple(LogField::Body), "new"); + } + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::Body)) + .as_deref(), + Some("new"), + ); + } + + #[test] + fn set_resource_attribute_is_noop() { + let mut record = record_with_body_and_attr("x", "k", "v"); + let resource = resource_with_attr("a", "b"); + let resource_before = resource.clone(); + { + let mut adapter = OtlpLogAdapter::new(&mut record, Some(&resource), None, None, None); + adapter.set_field( + &LogFieldSelector::ResourceAttribute(vec!["a".to_string()]), + "ignored", + ); + } + assert_eq!(resource, resource_before, "resource must not be mutated"); + } + + #[test] + fn delete_attribute_present() { + let mut record = record_with_body_and_attr("x", "user_id", "42"); + let removed = { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + }; + assert!(removed); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .is_none(), + ); + } + + #[test] + fn delete_attribute_absent_returns_false() { + let mut record = record_with_body_and_attr("x", "user_id", "42"); + let removed = { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["other".to_string()])) + }; + assert!(!removed); + } + + #[test] + fn delete_simple_body() { + let mut record = record_with_body_and_attr("x", "k", "v"); + let removed = { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.delete_field(&LogFieldSelector::Simple(LogField::Body)) + }; + assert!(removed); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::Body)) + .is_none(), + ); + } + + #[test] + fn move_attribute_within_log_attributes() { + let mut record = record_with_body_and_attr("x", "usr", "admin"); + { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["usr".to_string()]), + &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), + ); + } + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["usr".to_string()])) + .is_none(), + ); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .as_deref(), + Some("admin"), + ); + } +} diff --git a/src/transforms/policy/tests.rs b/src/transforms/policy/tests.rs index 2dddde088df1a..28f50392c56a1 100644 --- a/src/transforms/policy/tests.rs +++ b/src/transforms/policy/tests.rs @@ -9,16 +9,18 @@ use std::io::Write; use std::path::Path; use std::time::Duration; +use serde_json::json; use tempfile::NamedTempFile; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use vector_lib::config::LogNamespace; use vector_lib::event::{LogEvent, Metric, MetricKind, MetricValue}; use crate::event::{Event, Value}; use crate::test_util::components::init_test; use crate::transforms::test::create_topology; -use super::config::{PolicyConfig, PolicyProviderConfig}; +use super::config::{PolicyConfig, PolicyMode, PolicyProviderConfig}; use super::field_mapping::FieldMapping; /// Write `body` to a fresh NamedTempFile with the `.json` suffix and return @@ -41,6 +43,19 @@ fn policy_config(path: &Path) -> PolicyConfig { "local", path.to_string_lossy().into_owned(), )], + mode: PolicyMode::Flat, + field_mapping: FieldMapping::default(), + } +} + +#[allow(dead_code)] +fn policy_config_otel(path: &Path) -> PolicyConfig { + PolicyConfig { + policy_providers: vec![PolicyProviderConfig::file( + "local", + path.to_string_lossy().into_owned(), + )], + mode: PolicyMode::Otel, field_mapping: FieldMapping::default(), } } @@ -1409,3 +1424,535 @@ async fn rate_limit_per_minute_first_event_passes() { drop(tx); topology.stop().await; } + +// ============================================================================= +// OTel envelope-mode tests. +// +// These exercise the full envelope iteration path (resourceLogs[]→scopeLogs[] +// →logRecords[]) — per-record filtering, pruning of empty children, and +// dropping the whole event when all records are filtered out. +// ============================================================================= + +/// Build a single OTLP record body wrapped as an AnyValue. +fn otlp_any_string(s: &str) -> serde_json::Value { + json!({ "stringValue": s }) +} + +/// Build a single OTLP attribute entry `{ key, value: { stringValue: ... } }`. +fn otlp_attr(key: &str, value: &str) -> serde_json::Value { + json!({ "key": key, "value": otlp_any_string(value) }) +} + +/// Build an OTLP envelope with a single resourceLogs+scopeLogs entry and +/// the supplied records inside. +fn otlp_envelope(records: Vec) -> Event { + Event::from_json_value( + json!({ + "resourceLogs": [ + { + "resource": { + "attributes": [ + otlp_attr("service.name", "test-service"), + ] + }, + "schemaUrl": "https://opentelemetry.io/schemas/test", + "scopeLogs": [ + { + "scope": { + "name": "test.scope", + "attributes": [ + otlp_attr("library", "test-lib"), + ] + }, + "schemaUrl": "https://opentelemetry.io/schemas/test", + "logRecords": records + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .expect("valid envelope") +} + +/// Build a minimal OTLP log record with the given body string. +fn otlp_record(body: &str) -> serde_json::Value { + json!({ + "severityText": "INFO", + "body": otlp_any_string(body), + "attributes": [] + }) +} + +const OTEL_POLICY_DROP_DEBUG: &str = r#"{ + "policies": [ + { + "id": "drop-debug", + "name": "drop-debug", + "log": { + "match": [{ "log_field": "body", "regex": "debug" }], + "keep": "none" + } + } + ] +}"#; + +#[tokio::test] +async fn otel_mode_passes_non_envelope_event_through() { + // Backwards-compat: if someone configures `mode: otel` but routes a + // non-envelope (flat) event through it, we forward it unchanged + // rather than silently dropping data. + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let mut event = LogEvent::default(); + event.insert("message", "not an envelope"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("flat event passes through"); + assert_eq!( + received.into_log().get("message").and_then(|v| v.as_str()), + Some("not an envelope".into()), + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_drops_single_matching_record() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // The single record matches `debug`, so it's filtered out — leaving + // an empty logRecords → empty scopeLogs → empty resourceLogs → the + // whole event is dropped. + let envelope = otlp_envelope(vec![otlp_record("debug message")]); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_filters_some_records_keeps_envelope() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![ + otlp_record("info message"), + otlp_record("debug message"), + otlp_record("warn message"), + otlp_record("another debug here"), + ]); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + // Drill into resourceLogs[0].scopeLogs[0].logRecords and verify only + // the two non-debug records remained. + let records = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .and_then(|sl| sl.first()) + .and_then(|sl| sl.get("logRecords")) + .and_then(|v| v.as_array()) + .expect("records array still present"); + assert_eq!(records.len(), 2); + + let bodies: Vec = records + .iter() + .filter_map(|r| { + r.get("body") + .and_then(|b| b.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()) + }) + .collect(); + assert_eq!(bodies, vec!["info message", "warn message"]); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_prunes_empty_scope_logs() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // Two scopeLogs entries: the first has only matching records, the + // second has a non-matching record. The first entry should be pruned. + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "scopeLogs": [ + { + "scope": { "name": "scope-a" }, + "logRecords": [ + otlp_record("debug 1"), + otlp_record("debug 2"), + ] + }, + { + "scope": { "name": "scope-b" }, + "logRecords": [ + otlp_record("info kept"), + ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let scope_logs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .expect("scopeLogs still present"); + assert_eq!(scope_logs.len(), 1, "empty scope must be pruned"); + let scope_name = scope_logs[0] + .get("scope") + .and_then(|s| s.get("name")) + .and_then(|n| n.as_str()) + .map(|s| s.into_owned()); + assert_eq!(scope_name.as_deref(), Some("scope-b")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_prunes_empty_resource_logs() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // Two resourceLogs entries. The first ends up empty after filtering; + // the second still has a record. Expect: only the second remains. + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "resource": { "attributes": [ otlp_attr("service.name", "dropped") ] }, + "scopeLogs": [ + { + "scope": { "name": "s" }, + "logRecords": [ otlp_record("debug a") ] + } + ] + }, + { + "resource": { "attributes": [ otlp_attr("service.name", "kept") ] }, + "scopeLogs": [ + { + "scope": { "name": "s" }, + "logRecords": [ otlp_record("info b") ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let resource_logs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .expect("resourceLogs present"); + assert_eq!( + resource_logs.len(), + 1, + "empty resource entry must be pruned" + ); + + let service = resource_logs[0] + .get("resource") + .and_then(|r| r.get("attributes")) + .and_then(|a| a.as_array()) + .and_then(|a| a.first()) + .and_then(|kv| kv.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(service.as_deref(), Some("kept")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_resource_attribute_matcher_drops_record() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-service", + "name": "drop-by-service", + "log": { + "match": [{ "resource_attribute": "service.name", "exact": "noisy-service" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "resource": { "attributes": [ otlp_attr("service.name", "noisy-service") ] }, + "scopeLogs": [ + { + "scope": { "name": "s" }, + "logRecords": [ otlp_record("hello") ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_scope_attribute_matcher_drops_record() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-lib", + "name": "drop-by-lib", + "log": { + "match": [{ "scope_attribute": "library", "exact": "deprecated-tracer" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "scopeLogs": [ + { + "scope": { "attributes": [ otlp_attr("library", "deprecated-tracer") ] }, + "logRecords": [ otlp_record("hello") ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_redact_mutates_attribute_array() { + let policy = r#"{ + "policies": [ + { + "id": "redact-secret", + "name": "redact-secret", + "log": { + "match": [{ "log_field": "body", "regex": "login" }], + "keep": "all", + "transform": { + "redact": [ + { "log_attribute": "password", "replacement": "[REDACTED]" } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![json!({ + "severityText": "INFO", + "body": otlp_any_string("user login attempt"), + "attributes": [ + otlp_attr("password", "super-secret"), + otlp_attr("user", "alice"), + ] + })]); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let attrs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .and_then(|sl| sl.first()) + .and_then(|sl| sl.get("logRecords")) + .and_then(|v| v.as_array()) + .and_then(|r| r.first()) + .and_then(|r| r.get("attributes")) + .and_then(|v| v.as_array()) + .expect("attributes array present"); + + let password = attrs + .iter() + .find(|a| a.get("key").and_then(|k| k.as_str()).as_deref() == Some("password")); + let password_val = password + .and_then(|a| a.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(password_val.as_deref(), Some("[REDACTED]")); + + // Untouched attribute stays put. + let user_val = attrs + .iter() + .find(|a| a.get("key").and_then(|k| k.as_str()).as_deref() == Some("user")) + .and_then(|a| a.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(user_val.as_deref(), Some("alice")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_add_inserts_attribute_into_array() { + let policy = r#"{ + "policies": [ + { + "id": "tag-processed", + "name": "tag-processed", + "log": { + "match": [{ "log_field": "body", "regex": ".+" }], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": false } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![otlp_record("anything")]); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let attrs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .and_then(|sl| sl.first()) + .and_then(|sl| sl.get("logRecords")) + .and_then(|v| v.as_array()) + .and_then(|r| r.first()) + .and_then(|r| r.get("attributes")) + .and_then(|v| v.as_array()) + .expect("attributes array present"); + + let processed_by = attrs + .iter() + .find(|a| a.get("key").and_then(|k| k.as_str()).as_deref() == Some("processed_by")) + .and_then(|a| a.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(processed_by.as_deref(), Some("vector")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_severity_text_matcher_drops_record() { + let policy = r#"{ + "policies": [ + { + "id": "drop-debug-severity", + "name": "drop-debug-severity", + "log": { + "match": [{ "log_field": "severity_text", "exact": "DEBUG" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![json!({ + "severityText": "DEBUG", + "body": otlp_any_string("anything"), + "attributes": [] + })]); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_metric_event_passes_through_untouched() { + // Same guarantee as flat mode: non-log signals are forwarded as-is. + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let metric = Event::from(Metric::new( + "test", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + )); + tx.send(metric).await.unwrap(); + let received = recv(&mut out).await.expect("metric passes through"); + assert!(matches!(received, Event::Metric(_))); + + drop(tx); + topology.stop().await; +} diff --git a/src/transforms/policy/transform.rs b/src/transforms/policy/transform.rs index 1078dc13e3fa2..5abf079548d4a 100644 --- a/src/transforms/policy/transform.rs +++ b/src/transforms/policy/transform.rs @@ -5,14 +5,16 @@ use std::sync::Arc; use async_stream::stream; use futures::{Stream, StreamExt}; -use policy_rs::{EvaluateResult, PolicyEngine, PolicyRegistry}; +use policy_rs::{EvaluateResult, PolicyEngine, PolicyRegistry, PolicySnapshot}; use vector_lib::transform::TaskTransform; -use crate::event::Event; +use crate::event::{Event, LogEvent}; use super::adapter::VectorLogAdapter; +use super::config::PolicyMode; use super::field_mapping::FieldMapping; use super::internal_events::{DropReason, emit_dropped}; +use super::otlp_adapter::evaluate_envelope; /// Per-task state for the `policy` transform. /// @@ -24,6 +26,7 @@ pub struct Policy { registry: Arc, engine: Arc, mapping: Arc, + mode: PolicyMode, } impl Policy { @@ -31,11 +34,13 @@ impl Policy { registry: Arc, engine: Arc, mapping: Arc, + mode: PolicyMode, ) -> Self { Self { registry, engine, mapping, + mode, } } } @@ -51,51 +56,32 @@ impl TaskTransform for Policy { let registry = Arc::clone(&self.registry); let engine = Arc::clone(&self.engine); let mapping = Arc::clone(&self.mapping); + let mode = self.mode; Box::pin(stream! { while let Some(event) = input_rx.next().await { match event { - Event::Log(mut log) => { + Event::Log(log) => { // Pull a fresh snapshot every event so live-reloaded // policies take effect on the next pass. Snapshots // are cheap (Arc clone of an immutable inner). let snapshot = registry.snapshot(); - let result = { - let mut adapter = VectorLogAdapter::new(&mut log, &mapping); - engine.evaluate_and_transform(&snapshot, &mut adapter).await - }; - match result { - Ok(EvaluateResult::NoMatch) - | Ok(EvaluateResult::Keep { .. }) - | Ok(EvaluateResult::Sample { keep: true, .. }) - | Ok(EvaluateResult::RateLimit { allowed: true, .. }) => { - yield Event::Log(log); - } - Ok(EvaluateResult::Drop { .. }) => { - emit_dropped(DropReason::PolicyDrop); - } - Ok(EvaluateResult::Sample { keep: false, .. }) => { - emit_dropped(DropReason::SampleRejected); + let outcome = match mode { + PolicyMode::Flat => { + evaluate_flat(&engine, &snapshot, &mapping, log).await } - Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { - emit_dropped(DropReason::RateLimited); - } - Err(error) => { - // Fail open: an evaluation error shouldn't - // silently drop telemetry. Log and pass the - // event through untouched. - error!( - message = "Policy evaluation failed; event passed through unchanged.", - %error, - ); - yield Event::Log(log); + PolicyMode::Otel => { + evaluate_envelope(&engine, &snapshot, log).await } + }; + if let Some(forwarded) = outcome { + yield Event::Log(forwarded); } } other => { // policy-rs only targets the log signal in this // integration. Metrics and traces are forwarded - // untouched. + // untouched regardless of mode. yield other; } } @@ -103,3 +89,44 @@ impl TaskTransform for Policy { }) } } + +/// Flat-mode evaluation: one Vector event = one log record. Returns +/// `Some(log)` to forward, `None` to drop. +async fn evaluate_flat( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + mapping: &FieldMapping, + mut log: LogEvent, +) -> Option { + let result = { + let mut adapter = VectorLogAdapter::new(&mut log, mapping); + engine.evaluate_and_transform(snapshot, &mut adapter).await + }; + match result { + Ok(EvaluateResult::NoMatch) + | Ok(EvaluateResult::Keep { .. }) + | Ok(EvaluateResult::Sample { keep: true, .. }) + | Ok(EvaluateResult::RateLimit { allowed: true, .. }) => Some(log), + Ok(EvaluateResult::Drop { .. }) => { + emit_dropped(DropReason::PolicyDrop); + None + } + Ok(EvaluateResult::Sample { keep: false, .. }) => { + emit_dropped(DropReason::SampleRejected); + None + } + Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { + emit_dropped(DropReason::RateLimited); + None + } + Err(error) => { + // Fail open: an evaluation error shouldn't silently drop + // telemetry. Log and pass the event through untouched. + error!( + message = "Policy evaluation failed; event passed through unchanged.", + %error, + ); + Some(log) + } + } +} From 75df5af9338be4c533b91e11b8ca9919643993ca Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Thu, 28 May 2026 14:36:43 -0400 Subject: [PATCH 4/9] metric and trace support --- lib/codecs/src/common/mod.rs | 2 + lib/codecs/src/common/otlp.rs | 170 ++++ lib/codecs/src/decoding/format/otlp.rs | 47 +- lib/codecs/src/encoding/format/otlp.rs | 10 +- lib/opentelemetry-proto/src/common.rs | 3 + .../collector/logs/v1/logs_service.proto | 2 - .../metrics/v1/metrics_service.proto | 2 - .../collector/trace/v1/trace_service.proto | 2 - .../proto/common/v1/common.proto | 79 +- .../opentelemetry/proto/logs/v1/logs.proto | 33 +- .../proto/metrics/v1/metrics.proto | 103 +- .../proto/resource/v1/resource.proto | 10 +- .../opentelemetry/proto/trace/v1/trace.proto | 127 ++- src/sources/opentelemetry/tests.rs | 49 + src/transforms/policy/mod.rs | 2 + src/transforms/policy/otlp_adapter.rs | 926 +++++++++--------- src/transforms/policy/otlp_metric_adapter.rs | 217 ++++ src/transforms/policy/otlp_trace_adapter.rs | 307 ++++++ src/transforms/policy/transform.rs | 35 +- 19 files changed, 1596 insertions(+), 530 deletions(-) create mode 100644 lib/codecs/src/common/otlp.rs create mode 100644 src/transforms/policy/otlp_metric_adapter.rs create mode 100644 src/transforms/policy/otlp_trace_adapter.rs diff --git a/lib/codecs/src/common/mod.rs b/lib/codecs/src/common/mod.rs index e2d14804f7d3b..adc32341bf5fb 100644 --- a/lib/codecs/src/common/mod.rs +++ b/lib/codecs/src/common/mod.rs @@ -1,3 +1,5 @@ //! A collection of common utility features used by both encoding and decoding logic. pub mod length_delimited; +#[cfg(feature = "opentelemetry")] +pub mod otlp; diff --git a/lib/codecs/src/common/otlp.rs b/lib/codecs/src/common/otlp.rs new file mode 100644 index 0000000000000..1f9d21c1f7b37 --- /dev/null +++ b/lib/codecs/src/common/otlp.rs @@ -0,0 +1,170 @@ +//! Shared helpers for converting OTLP trace/span identifiers between their +//! protobuf wire form (raw `bytes`) and their OTLP/JSON form (hex strings). +//! +//! The protobuf `LogRecord.trace_id` / `Span.span_id` etc. are `bytes`, but the +//! canonical OTLP/JSON representation (and what every other tool in the +//! ecosystem, including downstream policy evaluation, expects) is a lowercase +//! hex string. Vector decodes OTLP protobuf into a JSON-shaped event tree, so we +//! normalize these identifiers to hex on decode and convert them back to raw +//! bytes on encode. Keeping the in-memory representation canonical means a plain +//! JSON serializer round-trips correctly and matchers see the hex string they +//! expect. + +use bytes::Bytes; +use vrl::value::Value; + +/// OTLP identifier fields (proto3 JSON camelCase) that are `bytes` on the wire +/// but hex strings in OTLP/JSON. +const ID_FIELDS: [&str; 3] = ["traceId", "spanId", "parentSpanId"]; + +/// Recursively hex-encode every OTLP id field carried as raw bytes. Called +/// after decoding OTLP protobuf so the event tree holds canonical hex strings. +pub fn hex_encode_ids(value: &mut Value) { + walk(value, true); +} + +/// Recursively hex-decode every OTLP id field carried as a hex string back into +/// raw bytes. Called before encoding to OTLP protobuf. +pub fn hex_decode_ids(value: &mut Value) { + walk(value, false); +} + +fn walk(value: &mut Value, encode: bool) { + match value { + Value::Object(map) => { + for (key, child) in map.iter_mut() { + if ID_FIELDS.contains(&key.as_str()) { + convert(child, encode); + } else { + walk(child, encode); + } + } + } + Value::Array(items) => { + for item in items.iter_mut() { + walk(item, encode); + } + } + _ => {} + } +} + +fn convert(value: &mut Value, encode: bool) { + let Value::Bytes(bytes) = value else { + return; + }; + if encode { + *value = Value::Bytes(Bytes::from(hex_encode(bytes))); + } else if let Some(raw) = hex_decode(bytes) { + *value = Value::Bytes(Bytes::from(raw)); + } +} + +fn hex_encode(bytes: &[u8]) -> Vec { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = Vec::with_capacity(bytes.len() * 2); + for &b in bytes { + out.push(DIGITS[(b >> 4) as usize]); + out.push(DIGITS[(b & 0x0f) as usize]); + } + out +} + +fn hex_decode(bytes: &[u8]) -> Option> { + if bytes.len() % 2 != 0 { + return None; + } + let nibble = |c: u8| -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } + }; + let mut out = Vec::with_capacity(bytes.len() / 2); + for pair in bytes.chunks_exact(2) { + out.push((nibble(pair[0])? << 4) | nibble(pair[1])?); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use vrl::value::Value; + + fn obj(pairs: Vec<(&str, Value)>) -> Value { + Value::Object( + pairs + .into_iter() + .map(|(k, v)| (k.into(), v)) + .collect(), + ) + } + + #[test] + fn round_trips_span_id() { + let raw = Bytes::from_static(b"span0001"); + let mut tree = obj(vec![("spanId", Value::Bytes(raw.clone()))]); + hex_encode_ids(&mut tree); + assert_eq!( + tree.as_object().unwrap().get("spanId").unwrap(), + &Value::Bytes(Bytes::from_static(b"7370616e30303031")), + ); + hex_decode_ids(&mut tree); + assert_eq!( + tree.as_object().unwrap().get("spanId").unwrap(), + &Value::Bytes(raw), + ); + } + + #[test] + fn walks_nested_arrays() { + let mut tree = obj(vec![( + "resourceSpans", + Value::Array(vec![obj(vec![( + "spans", + Value::Array(vec![obj(vec![( + "traceId", + Value::Bytes(Bytes::from_static(&[0x12, 0x34])), + )])]), + )])]), + )]); + hex_encode_ids(&mut tree); + let trace_id = tree + .as_object() + .unwrap() + .get("resourceSpans") + .unwrap() + .as_array() + .unwrap()[0] + .as_object() + .unwrap() + .get("spans") + .unwrap() + .as_array() + .unwrap()[0] + .as_object() + .unwrap() + .get("traceId") + .unwrap(); + assert_eq!(trace_id, &Value::Bytes(Bytes::from_static(b"1234"))); + } + + #[test] + fn leaves_attribute_key_named_span_id_untouched() { + // A user attribute whose *key string* is "spanId" lives under the "key" + // field, not as an object field named spanId, so it must not be touched. + let mut tree = obj(vec![( + "attributes", + Value::Array(vec![obj(vec![ + ("key", Value::from("spanId")), + ("value", obj(vec![("stringValue", Value::from("not-an-id"))])), + ])]), + )]); + let before = tree.clone(); + hex_encode_ids(&mut tree); + assert_eq!(tree, before); + } +} diff --git a/lib/codecs/src/decoding/format/otlp.rs b/lib/codecs/src/decoding/format/otlp.rs index 3e81dec089dce..94ec875b2f3e6 100644 --- a/lib/codecs/src/decoding/format/otlp.rs +++ b/lib/codecs/src/decoding/format/otlp.rs @@ -163,20 +163,23 @@ impl Deserializer for OtlpDeserializer { for signal_type in &self.signals { match signal_type { OtlpSignalType::Logs => { - if let Ok(events) = self.logs_deserializer.parse(bytes.clone(), log_namespace) + if let Ok(mut events) = + self.logs_deserializer.parse(bytes.clone(), log_namespace) && let Some(Event::Log(log)) = events.first() && log.get(RESOURCE_LOGS_JSON_FIELD).is_some() { + normalize_ids(&mut events); return Ok(events); } } OtlpSignalType::Metrics => { - if let Ok(events) = self + if let Ok(mut events) = self .metrics_deserializer .parse(bytes.clone(), log_namespace) && let Some(Event::Log(log)) = events.first() && log.get(RESOURCE_METRICS_JSON_FIELD).is_some() { + normalize_ids(&mut events); return Ok(events); } } @@ -188,7 +191,8 @@ impl Deserializer for OtlpDeserializer { && log.get(RESOURCE_SPANS_JSON_FIELD).is_some() { // Convert the log event to a trace event by taking ownership - if let Some(Event::Log(log)) = events.pop() { + if let Some(Event::Log(mut log)) = events.pop() { + crate::common::otlp::hex_encode_ids(log.value_mut()); let trace_event = Event::Trace(log.into()); return Ok(smallvec![trace_event]); } @@ -201,6 +205,17 @@ impl Deserializer for OtlpDeserializer { } } +/// Hex-encode the OTLP id `bytes` fields (`traceId`/`spanId`/`parentSpanId`) of +/// every log event so the in-memory tree carries canonical OTLP/JSON hex +/// strings rather than raw bytes. +fn normalize_ids(events: &mut SmallVec<[Event; 1]>) { + for event in events.iter_mut() { + if let Event::Log(log) = event { + crate::common::otlp::hex_encode_ids(log.value_mut()); + } + } +} + #[cfg(test)] mod tests { use opentelemetry_proto::proto::{ @@ -231,6 +246,7 @@ mod tests { resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, + entity_refs: vec![], }), scope_logs: vec![ScopeLogs { scope: None, @@ -245,6 +261,7 @@ mod tests { trace_id: vec![], span_id: vec![], observed_time_unix_nano: 0, + event_name: String::new(), }], schema_url: String::new(), }], @@ -261,6 +278,7 @@ mod tests { resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, + entity_refs: vec![], }), scope_metrics: vec![ScopeMetrics { scope: None, @@ -268,6 +286,7 @@ mod tests { name: "test_metric".to_string(), description: String::new(), unit: String::new(), + metadata: vec![], data: None, }], schema_url: String::new(), @@ -285,6 +304,7 @@ mod tests { resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, + entity_refs: vec![], }), scope_spans: vec![ScopeSpans { scope: None, @@ -304,6 +324,7 @@ mod tests { links: vec![], dropped_links_count: 0, status: None, + flags: 0, }], schema_url: String::new(), }], @@ -341,28 +362,28 @@ mod tests { let span = spans.first().expect("should have at least one span"); - // Verify traceId - should be raw bytes (16 bytes for trace_id) + // Verify traceId - normalized to a lowercase hex string (OTLP/JSON form) let trace_id = span .get("traceId") - .and_then(|v| v.as_bytes()) - .expect("traceId should exist and be bytes"); + .and_then(|v| v.as_str()) + .expect("traceId should exist and be a hex string"); assert_eq!( trace_id.as_ref(), - &TEST_TRACE_ID, - "traceId should match the expected 16 bytes (0102030405060708090a0b0c0d0e0f10)" + "0102030405060708090a0b0c0d0e0f10", + "traceId should be the hex encoding of the 16 raw bytes" ); - // Verify spanId - should be raw bytes (8 bytes for span_id) + // Verify spanId - normalized to a lowercase hex string (OTLP/JSON form) let span_id = span .get("spanId") - .and_then(|v| v.as_bytes()) - .expect("spanId should exist and be bytes"); + .and_then(|v| v.as_str()) + .expect("spanId should exist and be a hex string"); assert_eq!( span_id.as_ref(), - &TEST_SPAN_ID, - "spanId should match the expected 8 bytes (0102030405060708)" + "0102030405060708", + "spanId should be the hex encoding of the 8 raw bytes" ); } diff --git a/lib/codecs/src/encoding/format/otlp.rs b/lib/codecs/src/encoding/format/otlp.rs index c3e8d325a8abc..b5c69bd902f81 100644 --- a/lib/codecs/src/encoding/format/otlp.rs +++ b/lib/codecs/src/encoding/format/otlp.rs @@ -97,7 +97,15 @@ impl OtlpSerializer { impl Encoder for OtlpSerializer { type Error = vector_common::Error; - fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + fn encode(&mut self, mut event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + // The in-memory tree carries trace/span ids as hex strings (canonical + // OTLP/JSON). Convert them back to raw `bytes` before protobuf encoding. + match &mut event { + Event::Log(log) => crate::common::otlp::hex_decode_ids(log.value_mut()), + Event::Trace(trace) => crate::common::otlp::hex_decode_ids(trace.value_mut()), + Event::Metric(_) => {} + } + // Determine which descriptor to use based on top-level OTLP fields // This handles events that were decoded with use_otlp_decoding enabled // The deserializer uses use_json_names: true, so fields are in camelCase diff --git a/lib/opentelemetry-proto/src/common.rs b/lib/opentelemetry-proto/src/common.rs index d2cb2876447dc..e41f90eb7a7b8 100644 --- a/lib/opentelemetry-proto/src/common.rs +++ b/lib/opentelemetry-proto/src/common.rs @@ -20,6 +20,9 @@ impl From for Value { .collect::>(), ), PBValue::KvlistValue(arr) => kv_list_into_value(arr.values), + // String-table reference (profiles signal). We don't carry the + // referenced string table here, so there's nothing to resolve. + PBValue::StringValueStrindex(_) => Value::Null, } } } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto index 8260d8aaeb821..8be5cf75ef177 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto @@ -28,8 +28,6 @@ option go_package = "go.opentelemetry.io/proto/otlp/collector/logs/v1"; // OpenTelemetry and an collector, or between an collector and a central collector (in this // case logs are sent/received to/from multiple Applications). service LogsService { - // For performance reasons, it is recommended to keep this RPC - // alive for the entire life of the application. rpc Export(ExportLogsServiceRequest) returns (ExportLogsServiceResponse) {} } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto index dd48f1ad3a168..bc0242844151b 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto @@ -28,8 +28,6 @@ option go_package = "go.opentelemetry.io/proto/otlp/collector/metrics/v1"; // instrumented with OpenTelemetry and a collector, or between a collector and a // central collector. service MetricsService { - // For performance reasons, it is recommended to keep this RPC - // alive for the entire life of the application. rpc Export(ExportMetricsServiceRequest) returns (ExportMetricsServiceResponse) {} } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto index d6fe67f9e553d..efbbedbe4545a 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto @@ -28,8 +28,6 @@ option go_package = "go.opentelemetry.io/proto/otlp/collector/trace/v1"; // OpenTelemetry and a collector, or between a collector and a central collector (in this // case spans are sent/received to/from multiple Applications). service TraceService { - // For performance reasons, it is recommended to keep this RPC - // alive for the entire life of the application. rpc Export(ExportTraceServiceRequest) returns (ExportTraceServiceResponse) {} } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto index f7ee8f2653cc4..58a911ad49851 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto @@ -22,7 +22,7 @@ option java_package = "io.opentelemetry.proto.common.v1"; option java_outer_classname = "CommonProto"; option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; -// AnyValue is used to represent any type of attribute value. AnyValue may contain a +// Represents any type of attribute value. AnyValue may contain a // primitive value such as a string or integer or it may contain an arbitrary nested // object containing arrays, key-value lists and primitives. message AnyValue { @@ -36,6 +36,17 @@ message AnyValue { ArrayValue array_value = 5; KeyValueList kvlist_value = 6; bytes bytes_value = 7; + // Reference to the string value in ProfilesDictionary.string_table. + // + // Note: This is currently used exclusively in the Profiling signal. + // Implementers of OTLP receivers for signals other than Profiling should + // treat the presence of this value as a non-fatal issue. + // Log an error or warning indicating an unexpected field intended for the + // Profiling signal and process the data as if this value were absent or + // empty, ignoring its semantic content for the non-Profiling signal. + // + // Status: [Development] + int32 string_value_strindex = 8; } } @@ -54,28 +65,90 @@ message ArrayValue { message KeyValueList { // A collection of key/value pairs of key-value pairs. The list may be empty (may // contain 0 elements). + // // The keys MUST be unique (it is not allowed to have more than one // value with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated KeyValue values = 1; } -// KeyValue is a key-value pair that is used to store Span attributes, Link +// Represents a key-value pair that is used to store Span attributes, Link // attributes, etc. message KeyValue { + // The key name of the pair. + // key_ref MUST NOT be set if key is used. string key = 1; + + // The value of the pair. AnyValue value = 2; + + // Reference to the string key in ProfilesDictionary.string_table. + // key MUST NOT be set if key_strindex is used. + // + // Note: This is currently used exclusively in the Profiling signal. + // Implementers of OTLP receivers for signals other than Profiling should + // treat the presence of this key as a non-fatal issue. + // Log an error or warning indicating an unexpected field intended for the + // Profiling signal and process the data as if this value were absent or + // empty, ignoring its semantic content for the non-Profiling signal. + // + // Status: [Development] + int32 key_strindex = 3; } // InstrumentationScope is a message representing the instrumentation scope information -// such as the fully qualified name and version. +// such as the fully qualified name and version. message InstrumentationScope { + // A name denoting the Instrumentation scope. // An empty instrumentation scope name means the name is unknown. string name = 1; + + // Defines the version of the instrumentation scope. + // An empty instrumentation scope version means the version is unknown. string version = 2; // Additional attributes that describe the scope. [Optional]. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated KeyValue attributes = 3; + + // The number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 4; } + +// A reference to an Entity. +// Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs. +// +// Status: [Development] +message EntityRef { + // The Schema URL, if known. This is the identifier of the Schema that the entity data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // + // This schema_url applies to the data in this message and to the Resource attributes + // referenced by id_keys and description_keys. + // TODO: discuss if we are happy with this somewhat complicated definition of what + // the schema_url applies to. + // + // This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs. + string schema_url = 1; + + // Defines the type of the entity. MUST not change during the lifetime of the entity. + // For example: "service" or "host". This field is required and MUST not be empty + // for valid entities. + string type = 2; + + // Attribute Keys that identify the entity. + // MUST not change during the lifetime of the entity. The Id must contain at least one attribute. + // These keys MUST exist in the containing {message}.attributes. + repeated string id_keys = 3; + + // Descriptive (non-identifying) attribute keys of the entity. + // MAY change over the lifetime of the entity. MAY be empty. + // These attribute keys are not part of entity's identity. + // These keys MUST exist in the containing {message}.attributes. + repeated string description_keys = 4; +} \ No newline at end of file diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto index 0b4b649729c9e..a6597d1e1148e 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto @@ -55,6 +55,10 @@ message ResourceLogs { // A list of ScopeLogs that originate from a resource. repeated ScopeLogs scope_logs = 2; + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_logs" field which have their own schema_url field. string schema_url = 3; @@ -70,13 +74,17 @@ message ScopeLogs { // A list of log records. repeated LogRecord log_records = 2; - // This schema_url applies to all logs in the "logs" field. + // The Schema URL, if known. This is the identifier of the Schema that the log data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "scope" field and all logs in the + // "log_records" field. string schema_url = 3; } // Possible values for LogRecord.SeverityNumber. enum SeverityNumber { - // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used. SEVERITY_NUMBER_UNSPECIFIED = 0; SEVERITY_NUMBER_TRACE = 1; SEVERITY_NUMBER_TRACE2 = 2; @@ -104,9 +112,11 @@ enum SeverityNumber { SEVERITY_NUMBER_FATAL4 = 24; } -// LogRecordFlags is defined as a protobuf 'uint32' type and is to be used as -// bit-fields. Each non-zero value defined in this enum is a bit-mask. -// To extract the bit-field, for example, use an expression like: +// LogRecordFlags represents constants used to interpret the +// LogRecord.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: // // (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK) // @@ -164,6 +174,7 @@ message LogRecord { // Additional attributes that describe the specific event occurrence. [Optional]. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 6; uint32 dropped_attributes_count = 7; @@ -200,4 +211,16 @@ message LogRecord { // - the field is not present, // - the field contains an invalid value. bytes span_id = 10; + + // A unique identifier of event category/type. + // All events with the same event_name are expected to conform to the same + // schema for both their attributes and their body. + // + // Recommended to be fully qualified and short (no longer than 256 characters). + // + // Presence of event_name on the log record identifies this record + // as an event. + // + // [Optional]. + string event_name = 12; } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto index e85af8c3eae00..a6fab4ee750f9 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto @@ -29,6 +29,24 @@ option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1"; // storage, OR can be embedded by other protocols that transfer OTLP metrics // data but do not implement the OTLP protocol. // +// MetricsData +// └─── ResourceMetrics +// ├── Resource +// ├── SchemaURL +// └── ScopeMetrics +// ├── Scope +// ├── SchemaURL +// └── Metric +// ├── Name +// ├── Description +// ├── Unit +// └── data +// ├── Gauge +// ├── Sum +// ├── Histogram +// ├── ExponentialHistogram +// └── Summary +// // The main difference between this message and collector protocol is that // in this message there will not be any "control" or "metadata" specific to // OTLP protocol. @@ -55,6 +73,10 @@ message ResourceMetrics { // A list of metrics that originate from a resource. repeated ScopeMetrics scope_metrics = 2; + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_metrics" field which have their own schema_url field. string schema_url = 3; @@ -70,7 +92,12 @@ message ScopeMetrics { // A list of metrics that originate from an instrumentation library. repeated Metric metrics = 2; - // This schema_url applies to all metrics in the "metrics" field. + // The Schema URL, if known. This is the identifier of the Schema that the metric data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "scope" field and all metrics in the + // "metrics" field. string schema_url = 3; } @@ -79,7 +106,6 @@ message ScopeMetrics { // // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md // -// // The data model and relation between entities is shown in the // diagram below. Here, "DataPoint" is the term used to refer to any // one of the specific data point value types, and "points" is the term used @@ -91,7 +117,7 @@ message ScopeMetrics { // - DataPoint contains timestamps, attributes, and one of the possible value type // fields. // -// Metric +// Metric // +------------+ // |name | // |description | @@ -162,18 +188,18 @@ message ScopeMetrics { message Metric { reserved 4, 6, 8; - // name of the metric, including its DNS name prefix. It must be unique. + // The name of the metric. string name = 1; - // description of the metric, which can be used in documentation. + // A description of the metric, which can be used in documentation. string description = 2; - // unit in which the metric value is reported. Follows the format - // described by http://unitsofmeasure.org/ucum.html. + // The unit in which the metric value is reported. Follows the format + // described by https://unitsofmeasure.org/ucum.html. string unit = 3; // Data determines the aggregation type (if any) of the metric, what is the - // reported value type for the data points, as well as the relationship to + // reported value type for the data points, as well as the relatationship to // the time interval over which they are reported. oneof data { Gauge gauge = 5; @@ -182,6 +208,16 @@ message Metric { ExponentialHistogram exponential_histogram = 10; Summary summary = 11; } + + // Additional metadata attributes that describe the metric. [Optional]. + // Attributes are non-identifying. + // Consumers SHOULD NOT need to be aware of these attributes. + // These attributes MAY be used to encode information allowing + // for lossless roundtrip translation to / from another data model. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue metadata = 12; } // Gauge represents the type of a scalar metric that always exports the @@ -194,25 +230,31 @@ message Metric { // AggregationTemporality is not included. Consequently, this also means // "StartTimeUnixNano" is ignored for all data points. message Gauge { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). repeated NumberDataPoint data_points = 1; } // Sum represents the type of a scalar metric that is calculated as a sum of all // reported measurements over a time interval. message Sum { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). repeated NumberDataPoint data_points = 1; // aggregation_temporality describes if the aggregator reports delta changes // since last report time, or cumulative changes since a fixed start time. AggregationTemporality aggregation_temporality = 2; - // If "true" means that the sum is monotonic. + // Represents whether the sum is monotonic. bool is_monotonic = 3; } // Histogram represents the type of a metric that is calculated by aggregating // as a Histogram of all reported measurements over a time interval. message Histogram { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). repeated HistogramDataPoint data_points = 1; // aggregation_temporality describes if the aggregator reports delta changes @@ -223,6 +265,8 @@ message Histogram { // ExponentialHistogram represents the type of a metric that is calculated by aggregating // as a ExponentialHistogram of all reported double measurements over a time interval. message ExponentialHistogram { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). repeated ExponentialHistogramDataPoint data_points = 1; // aggregation_temporality describes if the aggregator reports delta changes @@ -232,11 +276,16 @@ message ExponentialHistogram { // Summary metric data are used to convey quantile summaries, // a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary) -// and OpenMetrics (see: https://github.com/OpenObservability/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) +// and OpenMetrics (see: https://github.com/prometheus/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) // data type. These data points cannot always be merged in a meaningful way. // While they can be useful in some applications, histogram data points are // recommended for new applications. +// Summary metrics do not have an aggregation temporality field. This is +// because the count and sum fields of a SummaryDataPoint are assumed to be +// cumulative values. message Summary { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). repeated SummaryDataPoint data_points = 1; } @@ -340,6 +389,7 @@ message NumberDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -388,6 +438,7 @@ message HistogramDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -415,7 +466,7 @@ message HistogramDataPoint { // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#histogram + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram optional double sum = 5; // bucket_counts is an optional field contains the count values of histogram @@ -424,7 +475,9 @@ message HistogramDataPoint { // The sum of the bucket_counts must equal the value in the count field. // // The number of elements in bucket_counts array must be by one greater than - // the number of elements in explicit_bounds array. + // the number of elements in explicit_bounds array. The exception to this rule + // is when the length of bucket_counts is 0, then the length of explicit_bounds + // must also be 0. repeated fixed64 bucket_counts = 6; // explicit_bounds specifies buckets with explicitly defined bounds for values. @@ -440,6 +493,9 @@ message HistogramDataPoint { // Histogram buckets are inclusive of their upper boundary, except the last // bucket where the boundary is at infinity. This format is intentionally // compatible with the OpenMetrics histogram definition. + // + // If bucket_counts length is 0 then explicit_bounds length must also be 0, + // otherwise the data point is invalid. repeated double explicit_bounds = 7; // (Optional) List of exemplars collected from @@ -467,6 +523,7 @@ message ExponentialHistogramDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -482,19 +539,19 @@ message ExponentialHistogramDataPoint { // 1970. fixed64 time_unix_nano = 3; - // count is the number of values in the population. Must be + // The number of values in the population. Must be // non-negative. This value must be equal to the sum of the "bucket_counts" // values in the positive and negative Buckets plus the "zero_count" field. fixed64 count = 4; - // sum of the values in the population. If count is zero then this field + // The sum of the values in the population. If count is zero then this field // must be zero. // // Note: Sum should only be filled out when measuring non-negative discrete // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#histogram + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram optional double sum = 5; // scale describes the resolution of the histogram. Boundaries are @@ -514,7 +571,7 @@ message ExponentialHistogramDataPoint { // values depend on the range of the data. sint32 scale = 6; - // zero_count is the count of values that are either exactly zero or + // The count of values that are either exactly zero or // within the region considered zero by the instrumentation at the // tolerated degree of precision. This bucket stores values that // cannot be expressed using the standard exponential formula as @@ -533,12 +590,12 @@ message ExponentialHistogramDataPoint { // Buckets are a set of bucket counts, encoded in a contiguous array // of counts. message Buckets { - // Offset is the bucket index of the first entry in the bucket_counts array. + // The bucket index of the first entry in the bucket_counts array. // // Note: This uses a varint encoding as a simple form of compression. sint32 offset = 1; - // bucket_counts is an array of count values, where bucket_counts[i] carries + // An array of count values, where bucket_counts[i] carries // the count of the bucket at index (offset+i). bucket_counts[i] is the count // of values greater than base^(offset+i) and less than or equal to // base^(offset+i+1). @@ -558,10 +615,10 @@ message ExponentialHistogramDataPoint { // measurements that were used to form the data point repeated Exemplar exemplars = 11; - // min is the minimum value over (start_time, end_time]. + // The minimum value over (start_time, end_time]. optional double min = 12; - // max is the maximum value over (start_time, end_time]. + // The maximum value over (start_time, end_time]. optional double max = 13; // ZeroThreshold may be optionally set to convey the width of the zero @@ -574,7 +631,8 @@ message ExponentialHistogramDataPoint { } // SummaryDataPoint is a single data point in a timeseries that describes the -// time-varying values of a Summary metric. +// time-varying values of a Summary metric. The count and sum fields represent +// cumulative values. message SummaryDataPoint { reserved 1; @@ -582,6 +640,7 @@ message SummaryDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -607,7 +666,7 @@ message SummaryDataPoint { // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#summary + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary double sum = 5; // Represents the value at a given quantile of a distribution. diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto index 6637560bc3544..42c5913cfae75 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto @@ -29,9 +29,17 @@ message Resource { // Set of attributes that describe the resource. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - // dropped_attributes_count is the number of dropped attributes. If the value is 0, then + // The number of dropped attributes. If the value is 0, then // no attributes were dropped. uint32 dropped_attributes_count = 2; + + // Set of entities that participate in this Resource. + // + // Note: keys in the references MUST exist in attributes of this message. + // + // Status: [Development] + repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3; } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto index 864253ee41d87..8a992c148c642 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto @@ -55,6 +55,10 @@ message ResourceSpans { // A list of ScopeSpans that originate from a resource. repeated ScopeSpans scope_spans = 2; + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_spans" field which have their own schema_url field. string schema_url = 3; @@ -70,7 +74,12 @@ message ScopeSpans { // A list of Spans that originate from an instrumentation scope. repeated Span spans = 2; - // This schema_url applies to all spans and span events in the "spans" field. + // The Schema URL, if known. This is the identifier of the Schema that the span data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "scope" field and all spans and span + // events in the "spans" field. string schema_url = 3; } @@ -103,6 +112,29 @@ message Span { // field must be empty. The ID is an 8-byte array. bytes parent_span_id = 4; + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether a span's parent + // is remote. The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // When creating span messages, if the message is logically forwarded from another source + // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + // be copied as-is. If creating from a source that does not have an equivalent flags field + // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + // be set to zero. + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // + // [Optional]. + fixed32 flags = 16; + // A description of the span's operation. // // For example, the name can be a qualified method name or a file name @@ -151,7 +183,7 @@ message Span { // and `SERVER` (callee) to identify queueing latency associated with the span. SpanKind kind = 6; - // start_time_unix_nano is the start time of the span. On the client side, this is the time + // The start time of the span. On the client side, this is the time // kept by the local machine where the span execution starts. On the server side, this // is the time when the server's application handler starts running. // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. @@ -159,7 +191,7 @@ message Span { // This field is semantically required and it is expected that end_time >= start_time. fixed64 start_time_unix_nano = 7; - // end_time_unix_nano is the end time of the span. On the client side, this is the time + // The end time of the span. On the client side, this is the time // kept by the local machine where the span execution ends. On the server side, this // is the time when the server application handler stops running. // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. @@ -167,21 +199,20 @@ message Span { // This field is semantically required and it is expected that end_time >= start_time. fixed64 end_time_unix_nano = 8; - // attributes is a collection of key/value pairs. Note, global attributes + // A collection of key/value pairs. Note, global attributes // like server name can be set using the resource API. Examples of attributes: // - // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" - // "/http/server_latency": 300 - // "example.com/my_attribute": true - // "example.com/score": 10.239 + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "example.com/myattribute": true + // "example.com/score": 10.239 // - // The OpenTelemetry API specification further restricts the allowed value types: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; - // dropped_attributes_count is the number of attributes that were discarded. Attributes + // The number of attributes that were discarded. Attributes // can be discarded because their keys are too long or because there are too many // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 10; @@ -189,27 +220,28 @@ message Span { // Event is a time-stamped annotation of the span, consisting of user-supplied // text description and key-value pairs. message Event { - // time_unix_nano is the time the event occurred. + // The time the event occurred. fixed64 time_unix_nano = 1; - // name of the event. + // The name of the event. // This field is semantically required to be set to non-empty string. string name = 2; - // attributes is a collection of attribute key/value pairs on the event. + // A collection of attribute key/value pairs on the event. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; - // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // The number of dropped attributes. If the value is 0, // then no attributes were dropped. uint32 dropped_attributes_count = 4; } - // events is a collection of Event items. + // A collection of Event items. repeated Event events = 11; - // dropped_events_count is the number of dropped events. If the value is 0, then no + // The number of dropped events. If the value is 0, then no // events were dropped. uint32 dropped_events_count = 12; @@ -228,21 +260,41 @@ message Span { // The trace_state associated with the link. string trace_state = 3; - // attributes is a collection of attribute key/value pairs on the link. + // A collection of attribute key/value pairs on the link. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; - // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // The number of dropped attributes. If the value is 0, // then no attributes were dropped. uint32 dropped_attributes_count = 5; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether the link is remote. + // The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + // + // [Optional]. + fixed32 flags = 6; } - // links is a collection of Links, which are references from this span to a span + // A collection of Links, which are references from this span to a span // in the same or different trace. repeated Link links = 13; - // dropped_links_count is the number of dropped links after the maximum size was + // The number of dropped links after the maximum size was // enforced. If this value is 0, then no links were dropped. uint32 dropped_links_count = 14; @@ -264,7 +316,7 @@ message Status { enum StatusCode { // The default status. STATUS_CODE_UNSET = 0; - // The Span has been validated by an Application developer or Operator to + // The Span has been validated by an Application developer or Operator to // have completed successfully. STATUS_CODE_OK = 1; // The Span contains an error. @@ -274,3 +326,34 @@ message Status { // The status code. StatusCode code = 3; } + +// SpanFlags represents constants used to interpret the +// Span.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: +// +// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) +// +// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. +// +// Note that Span flags were introduced in version 1.1 of the +// OpenTelemetry protocol. Older Span producers do not set this +// field, consequently consumers should not rely on the absence of a +// particular flag bit to indicate the presence of a particular feature. +enum SpanFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + SPAN_FLAGS_DO_NOT_USE = 0; + + // Bits 0-7 are used for trace flags. + SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; + + // Bits 8 and 9 are used to indicate that the parent span or link span is remote. + // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; + + // Bits 10-31 are reserved for future use. +} diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index 9fb0bac71a1cd..10c5bfde42745 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -56,8 +56,10 @@ fn create_test_logs_request() -> Request { Request::new(ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { resource: Some(OtelResource { + entity_refs: vec![], attributes: vec![KeyValue { key: "res_key".into(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("res_val".into())), }), @@ -70,6 +72,7 @@ fn create_test_logs_request() -> Request { version: "1.2.3".into(), attributes: vec![KeyValue { key: "scope_attr".into(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("scope_val".into())), }), @@ -79,6 +82,7 @@ fn create_test_logs_request() -> Request { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, + event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { @@ -86,6 +90,7 @@ fn create_test_logs_request() -> Request { }), attributes: vec![KeyValue { key: "attr_key".into(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("attr_val".into())), }), @@ -107,8 +112,10 @@ fn create_test_metrics_request() -> ExportMetricsServiceRequest { ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -128,17 +135,20 @@ fn create_test_metrics_request() -> ExportMetricsServiceRequest { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::Summary(Summary { data_points: vec![SummaryDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -191,6 +201,7 @@ fn create_test_traces_request() -> ExportTraceServiceRequest { dropped_events_count: 0, links: vec![], dropped_links_count: 0, + flags: 0, status: None, trace_state: "".to_string(), }], @@ -382,8 +393,10 @@ async fn receive_sum_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -403,16 +416,19 @@ async fn receive_sum_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::Sum(Sum { data_points: vec![NumberDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -474,8 +490,10 @@ async fn receive_sum_non_monotonic_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -495,16 +513,19 @@ async fn receive_sum_non_monotonic_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::Sum(Sum { data_points: vec![NumberDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -566,8 +587,10 @@ async fn receive_gauge_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -587,16 +610,19 @@ async fn receive_gauge_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::Gauge(Gauge { data_points: vec![NumberDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -655,8 +681,10 @@ async fn receive_histogram_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -676,18 +704,21 @@ async fn receive_histogram_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::Histogram(Histogram { aggregation_temporality: AggregationTemporality::Cumulative as i32, data_points: vec![HistogramDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -782,8 +813,10 @@ async fn receive_histogram_delta_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -803,18 +836,21 @@ async fn receive_histogram_delta_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::Histogram(Histogram { aggregation_temporality: AggregationTemporality::Delta as i32, data_points: vec![HistogramDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -909,8 +945,10 @@ async fn receive_expontential_histogram_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -930,18 +968,21 @@ async fn receive_expontential_histogram_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::ExponentialHistogram(ExponentialHistogram { aggregation_temporality: AggregationTemporality::Cumulative as i32, data_points: vec![ExponentialHistogramDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -1049,8 +1090,10 @@ async fn receive_summary_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { + entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -1070,17 +1113,20 @@ async fn receive_summary_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), + metadata: vec![], data: Some(Data::Summary(Summary { data_points: vec![SummaryDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), + key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -1254,6 +1300,7 @@ async fn http_headers_logs_use_otlp_decoding_false() { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, + event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { @@ -1335,6 +1382,7 @@ async fn http_headers_logs_use_otlp_decoding_true() { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, + event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { @@ -1618,6 +1666,7 @@ async fn http_logs_use_otlp_decoding_emits_metric() { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, + event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { diff --git a/src/transforms/policy/mod.rs b/src/transforms/policy/mod.rs index e86975f977601..9969d9ed806ff 100644 --- a/src/transforms/policy/mod.rs +++ b/src/transforms/policy/mod.rs @@ -10,6 +10,8 @@ mod adapter; mod field_mapping; mod internal_events; mod otlp_adapter; +mod otlp_metric_adapter; +mod otlp_trace_adapter; pub mod config; pub mod transform; diff --git a/src/transforms/policy/otlp_adapter.rs b/src/transforms/policy/otlp_adapter.rs index 97fb065dc97c8..bdfd4ff064d14 100644 --- a/src/transforms/policy/otlp_adapter.rs +++ b/src/transforms/policy/otlp_adapter.rs @@ -7,23 +7,27 @@ //! it through `policy-rs` via [`OtlpLogAdapter`], filters in place, and //! prunes empty `scopeLogs` / `resourceLogs` entries. //! -//! Differences from [`super::adapter::VectorLogAdapter`]: +//! The adapter mirrors the conformance reference adapter +//! (`policy-conformance/runners/rs/src/eval.rs`) so behaviour matches the +//! spec exactly: //! //! * Field names are camelCase per the proto3 JSON mapping //! (`severityText`, not `severity_text`). -//! * `body` is an OTLP `AnyValue`, so `body.stringValue` (and other -//! variants) is the real path to the value. -//! * Attributes are always arrays of `{ key, value: AnyValue }` — not -//! key-value maps. Lookup is a linear scan by key. -//! * Resource and scope are read-only in this mode: mutating cloned -//! siblings would not affect the underlying envelope, so -//! `set_field` / `delete_field` / `move_field` on -//! `ResourceAttribute` / `ScopeAttribute` selectors are no-ops. We -//! can lift this restriction later by restructuring the iteration if -//! real users need it. -//! * Multi-segment `LogAttribute(["http", "method"])` paths are -//! dot-joined and looked up as the single OTel key `"http.method"`. -//! Walking `kvlistValue` for true nesting is a future extension. +//! * `body` is an OTLP `AnyValue`; only a non-empty `stringValue` is +//! matchable. Non-string variants (`intValue`, `boolValue`, …) are not +//! coerced to strings for matching, so a regex redact targeting a string +//! never mutates them. +//! * Simple string fields (`severityText`, `traceId`, `spanId`, +//! `eventName`, schema URLs) treat the empty string as absent. +//! * `traceId` / `spanId` arrive already hex-encoded (the OTLP decoder +//! normalizes the raw `bytes` to canonical OTLP/JSON hex), so they are +//! plain strings here. +//! * Attributes are arrays of `{ key, value: AnyValue }`. Multi-segment +//! selectors (`["http", "method"]`) walk nested `kvlistValue` entries. +//! * Resource and scope attributes are mutable: the envelope iteration +//! lends the adapter `&mut` access to the parent `resource` / `scope` +//! objects, so add / redact / remove / rename on +//! `ResourceAttribute` / `ScopeAttribute` selectors mutate them in place. use std::borrow::Cow; @@ -57,10 +61,13 @@ pub(super) async fn evaluate_envelope( let mut i = 0; while i < resource_logs.len() { - // Clone the resource sub-object and the resourceLogs entry's - // schemaUrl so the adapter can hold immutable refs to them without - // aliasing the mutable borrow we'll take of `scopeLogs` below. - let resource = resource_logs[i].get("resource").cloned(); + // Lift `resource` out of the entry so the adapter can hold a mutable + // borrow of it alongside the mutable borrow of `scopeLogs` we take + // below (the borrow checker can't prove those paths are disjoint). + // It is re-inserted after all records under this resource are done. + let mut resource = resource_logs[i] + .as_object_mut() + .and_then(|o| o.remove("resource")); let resource_schema_url = resource_logs[i].get("schemaUrl").cloned(); let mut prune_this_rl = false; @@ -71,7 +78,10 @@ pub(super) async fn evaluate_envelope( { let mut j = 0; while j < scope_logs.len() { - let scope = scope_logs[j].get("scope").cloned(); + // Same trick for `scope`. + let mut scope = scope_logs[j] + .as_object_mut() + .and_then(|o| o.remove("scope")); let scope_schema_url = scope_logs[j].get("schemaUrl").cloned(); let mut prune_this_sl = false; @@ -85,8 +95,8 @@ pub(super) async fn evaluate_envelope( let result = { let mut adapter = OtlpLogAdapter::new( &mut records[k], - resource.as_ref(), - scope.as_ref(), + resource.as_mut(), + scope.as_mut(), resource_schema_url.as_ref(), scope_schema_url.as_ref(), ); @@ -124,6 +134,13 @@ pub(super) async fn evaluate_envelope( prune_this_sl = records.is_empty(); } + // Re-attach the (possibly mutated) scope. + if let Some(scope) = scope + && let Some(obj) = scope_logs[j].as_object_mut() + { + obj.insert("scope".into(), scope); + } + if prune_this_sl { scope_logs.remove(j); } else { @@ -133,6 +150,13 @@ pub(super) async fn evaluate_envelope( prune_this_rl = scope_logs.is_empty(); } + // Re-attach the (possibly mutated) resource. + if let Some(resource) = resource + && let Some(obj) = resource_logs[i].as_object_mut() + { + obj.insert("resource".into(), resource); + } + if prune_this_rl { resource_logs.remove(i); } else { @@ -151,17 +175,17 @@ pub(super) async fn evaluate_envelope( /// and scope) to the `policy-rs` engine. pub(super) struct OtlpLogAdapter<'a> { log_record: &'a mut Value, - resource: Option<&'a Value>, - scope: Option<&'a Value>, + resource: Option<&'a mut Value>, + scope: Option<&'a mut Value>, resource_schema_url: Option<&'a Value>, scope_schema_url: Option<&'a Value>, } impl<'a> OtlpLogAdapter<'a> { - pub(super) const fn new( + pub(super) fn new( log_record: &'a mut Value, - resource: Option<&'a Value>, - scope: Option<&'a Value>, + resource: Option<&'a mut Value>, + scope: Option<&'a mut Value>, resource_schema_url: Option<&'a Value>, scope_schema_url: Option<&'a Value>, ) -> Self { @@ -174,31 +198,17 @@ impl<'a> OtlpLogAdapter<'a> { } } - /// Look up the value at a simple-field selector for reads. Returns the - /// `&Value` (still wrapped) so callers can choose between - /// stringification (for `get_field`) and presence-only checks (for - /// `field_exists`). - fn simple_value(&self, field: LogField) -> Option<&Value> { - match field { - LogField::Body => self.log_record.get("body"), - LogField::SeverityText => self.log_record.get("severityText"), - LogField::TraceId => self.log_record.get("traceId"), - LogField::SpanId => self.log_record.get("spanId"), - LogField::EventName => self.log_record.get("eventName"), - LogField::ResourceSchemaUrl => self.resource_schema_url, - LogField::ScopeSchemaUrl => self.scope_schema_url, - LogField::Unspecified => None, - } - } - - /// Look up the attributes array for an attribute-namespace selector. + /// Borrow the attributes array for an attribute-namespace selector. fn attributes_for(&self, selector: &LogFieldSelector) -> Option<&Value> { match selector { LogFieldSelector::LogAttribute(_) => self.log_record.get("attributes"), - LogFieldSelector::ResourceAttribute(_) => { - self.resource.and_then(|r| r.get("attributes")) + LogFieldSelector::ResourceAttribute(_) => self + .resource + .as_deref() + .and_then(|r| r.get("attributes")), + LogFieldSelector::ScopeAttribute(_) => { + self.scope.as_deref().and_then(|s| s.get("attributes")) } - LogFieldSelector::ScopeAttribute(_) => self.scope.and_then(|s| s.get("attributes")), LogFieldSelector::Simple(_) => None, } } @@ -210,30 +220,43 @@ impl Matchable for OtlpLogAdapter<'_> { fn get_field(&self, field: &LogFieldSelector) -> Option> { match field { LogFieldSelector::Simple(LogField::Body) => { - self.log_record.get("body").and_then(any_value_to_string) + any_value_string(self.log_record.get("body")) } - LogFieldSelector::Simple(simple) => { - self.simple_value(*simple).and_then(plain_value_to_string) + LogFieldSelector::Simple(LogField::SeverityText) => { + non_empty(self.log_record.get("severityText")) + } + LogFieldSelector::Simple(LogField::TraceId) => { + non_empty(self.log_record.get("traceId")) } + LogFieldSelector::Simple(LogField::SpanId) => non_empty(self.log_record.get("spanId")), + LogFieldSelector::Simple(LogField::EventName) => { + non_empty(self.log_record.get("eventName")) + } + LogFieldSelector::Simple(LogField::ResourceSchemaUrl) => { + non_empty(self.resource_schema_url) + } + LogFieldSelector::Simple(LogField::ScopeSchemaUrl) => non_empty(self.scope_schema_url), + LogFieldSelector::Simple(LogField::Unspecified) => None, LogFieldSelector::LogAttribute(path) | LogFieldSelector::ResourceAttribute(path) | LogFieldSelector::ScopeAttribute(path) => { - let attrs = self.attributes_for(field)?; - find_attribute_value(attrs, &join_path(path)) + find_attribute_path(self.attributes_for(field), path) } } } fn field_exists(&self, field: &LogFieldSelector) -> bool { match field { + LogFieldSelector::Simple(LogField::Body) => { + log_body_present(self.log_record.get("body")) + } LogFieldSelector::Simple(LogField::Unspecified) => false, - LogFieldSelector::Simple(simple) => self.simple_value(*simple).is_some(), + LogFieldSelector::Simple(_) => self.get_field(field).is_some(), LogFieldSelector::LogAttribute(path) | LogFieldSelector::ResourceAttribute(path) - | LogFieldSelector::ScopeAttribute(path) => match self.attributes_for(field) { - Some(attrs) => attribute_exists(attrs, &join_path(path)), - None => false, - }, + | LogFieldSelector::ScopeAttribute(path) => { + attribute_exists_path(self.attributes_for(field), path) + } } } } @@ -242,7 +265,9 @@ impl Transformable for OtlpLogAdapter<'_> { fn set_field(&mut self, field: &LogFieldSelector, value: &str) { match field { LogFieldSelector::Simple(LogField::Body) => { - self.log_record.insert("body", make_string_any_value(value)); + if let Some(obj) = self.log_record.as_object_mut() { + obj.insert("body".into(), make_string_any_value(value)); + } } LogFieldSelector::Simple(LogField::SeverityText) => { insert_simple_string(self.log_record, "severityText", value); @@ -256,162 +281,241 @@ impl Transformable for OtlpLogAdapter<'_> { LogFieldSelector::Simple(LogField::EventName) => { insert_simple_string(self.log_record, "eventName", value); } - // Simple fields whose backing storage isn't on the log record - // itself (schema URLs live on the parent entries) are read-only - // for the same reason ResourceAttribute / ScopeAttribute are: - // we only hold immutable refs. + // Schema URLs / unspecified have no log-record-local storage. LogFieldSelector::Simple(_) => {} LogFieldSelector::LogAttribute(path) => { - let key = join_path(path); - ensure_attributes_array(self.log_record); - if let Some(attrs) = self - .log_record - .get_mut("attributes") - .and_then(Value::as_array_mut) + if let Some(attrs) = ensure_attributes(self.log_record) { + set_string_attr(attrs, path, value); + } + } + LogFieldSelector::ResourceAttribute(path) => { + if let Some(resource) = self.resource.as_deref_mut() + && let Some(attrs) = ensure_attributes(resource) { - set_attribute(attrs, &key, value); + set_string_attr(attrs, path, value); } } - LogFieldSelector::ResourceAttribute(_) | LogFieldSelector::ScopeAttribute(_) => { - // Read-only in OTel mode; see module docstring. + LogFieldSelector::ScopeAttribute(path) => { + if let Some(scope) = self.scope.as_deref_mut() + && let Some(attrs) = ensure_attributes(scope) + { + set_string_attr(attrs, path, value); + } } } } fn delete_field(&mut self, field: &LogFieldSelector) -> bool { match field { - LogFieldSelector::Simple(LogField::Body) => self.log_record_remove("body"), + LogFieldSelector::Simple(LogField::Body) => remove_key(self.log_record, "body"), LogFieldSelector::Simple(LogField::SeverityText) => { - self.log_record_remove("severityText") + remove_key(self.log_record, "severityText") } - LogFieldSelector::Simple(LogField::TraceId) => self.log_record_remove("traceId"), - LogFieldSelector::Simple(LogField::SpanId) => self.log_record_remove("spanId"), - LogFieldSelector::Simple(LogField::EventName) => self.log_record_remove("eventName"), - LogFieldSelector::Simple(_) => false, - LogFieldSelector::LogAttribute(path) => { - let key = join_path(path); - self.log_record - .get_mut("attributes") - .and_then(Value::as_array_mut) - .map(|attrs| remove_attribute(attrs, &key)) - .unwrap_or(false) + LogFieldSelector::Simple(LogField::TraceId) => remove_key(self.log_record, "traceId"), + LogFieldSelector::Simple(LogField::SpanId) => remove_key(self.log_record, "spanId"), + LogFieldSelector::Simple(LogField::EventName) => { + remove_key(self.log_record, "eventName") } - LogFieldSelector::ResourceAttribute(_) | LogFieldSelector::ScopeAttribute(_) => false, + LogFieldSelector::Simple(_) => false, + LogFieldSelector::LogAttribute(path) => attributes_of_mut(self.log_record) + .map(|attrs| remove_attr(attrs, path)) + .unwrap_or(false), + LogFieldSelector::ResourceAttribute(path) => self + .resource + .as_deref_mut() + .and_then(attributes_of_mut) + .map(|attrs| remove_attr(attrs, path)) + .unwrap_or(false), + LogFieldSelector::ScopeAttribute(path) => self + .scope + .as_deref_mut() + .and_then(attributes_of_mut) + .map(|attrs| remove_attr(attrs, path)) + .unwrap_or(false), } } fn move_field(&mut self, from: &LogFieldSelector, to: &LogFieldSelector) { - // The engine guarantees `from` exists and `to` does not. In OTel - // mode we only support moves within the log-record's own - // attributes array. - let (LogFieldSelector::LogAttribute(from_path), LogFieldSelector::LogAttribute(to_path)) = - (from, to) - else { + // The engine guarantees `from` exists. Remove the underlying + // `{key, value}` entry (preserving the OTel value type) and re-insert + // it under `to`'s key in `to`'s namespace, overwriting any existing + // entry there (upsert semantics, matching the reference adapter). + let source = match from { + LogFieldSelector::LogAttribute(path) => { + attributes_of_mut(self.log_record).and_then(|attrs| remove_attr_kv(attrs, path)) + } + LogFieldSelector::ResourceAttribute(path) => self + .resource + .as_deref_mut() + .and_then(attributes_of_mut) + .and_then(|attrs| remove_attr_kv(attrs, path)), + LogFieldSelector::ScopeAttribute(path) => self + .scope + .as_deref_mut() + .and_then(attributes_of_mut) + .and_then(|attrs| remove_attr_kv(attrs, path)), + LogFieldSelector::Simple(_) => None, + }; + let Some(mut entry) = source else { return; }; - let from_key = join_path(from_path); - let to_key = join_path(to_path); - if let Some(attrs) = self - .log_record - .get_mut("attributes") - .and_then(Value::as_array_mut) - && let Some(idx) = attribute_index(attrs, &from_key) - { - let entry = attrs.remove(idx); - if let Some(value) = entry.as_object().and_then(|o| o.get("value")).cloned() { - attrs.push(make_attribute_entry(&to_key, value)); - } + + let target_key = match to { + LogFieldSelector::LogAttribute(path) + | LogFieldSelector::ResourceAttribute(path) + | LogFieldSelector::ScopeAttribute(path) => path.first().cloned(), + LogFieldSelector::Simple(_) => None, + }; + let Some(key) = target_key else { + return; + }; + if let Some(obj) = entry.as_object_mut() { + obj.insert("key".into(), Value::from(key.clone())); } - } -} -impl OtlpLogAdapter<'_> { - fn log_record_remove(&mut self, key: &str) -> bool { - match self.log_record.as_object_mut() { - Some(obj) => obj.remove(key).is_some(), - None => false, + match to { + LogFieldSelector::LogAttribute(_) => { + if let Some(attrs) = ensure_attributes(self.log_record) { + upsert_entry(attrs, &key, entry); + } + } + LogFieldSelector::ResourceAttribute(_) => { + if let Some(resource) = self.resource.as_deref_mut() + && let Some(attrs) = ensure_attributes(resource) + { + upsert_entry(attrs, &key, entry); + } + } + LogFieldSelector::ScopeAttribute(_) => { + if let Some(scope) = self.scope.as_deref_mut() + && let Some(attrs) = ensure_attributes(scope) + { + upsert_entry(attrs, &key, entry); + } + } + LogFieldSelector::Simple(_) => {} } } } // ============================================================================= -// Value-shape helpers. +// AnyValue helpers. // ============================================================================= -/// Coerce an OTLP `AnyValue` object to a string for matching purposes. -/// -/// `AnyValue` is a oneof with seven variants (`stringValue`, `intValue`, -/// `boolValue`, `doubleValue`, `bytesValue`, `arrayValue`, `kvlistValue`). -/// Scalar variants get stringified; container/binary variants return -/// `None` because there's no canonical text form for matching. -fn any_value_to_string(value: &Value) -> Option> { - let obj = value.as_object()?; - if let Some(v) = obj.get("stringValue") - && let Some(s) = v.as_str() - { - return Some(s); - } - if let Some(v) = obj.get("intValue") - && let Some(i) = v.as_integer() - { - return Some(Cow::Owned(i.to_string())); +/// Coerce an OTLP `AnyValue` to a string for matching. Only a non-empty +/// `stringValue` is matchable; all other variants (`intValue`, `boolValue`, +/// `arrayValue`, …) return `None` so string matchers and regex redaction +/// never operate on them. +fn any_value_string(value: Option<&Value>) -> Option> { + let obj = value?.as_object()?; + match obj.get("stringValue").and_then(Value::as_str) { + Some(s) if !s.is_empty() => Some(s), + _ => None, } - if let Some(v) = obj.get("boolValue") - && let Some(b) = v.as_boolean() - { - return Some(Cow::Borrowed(if b { "true" } else { "false" })); +} + +/// Whether an `AnyValue` carries any value variant at all. Powers +/// `exists: true` matchers for attributes whose value isn't a string. +fn any_value_present(value: Option<&Value>) -> bool { + const VARIANTS: [&str; 7] = [ + "stringValue", + "boolValue", + "intValue", + "doubleValue", + "arrayValue", + "kvlistValue", + "bytesValue", + ]; + match value.and_then(Value::as_object) { + Some(obj) => VARIANTS.iter().any(|k| obj.get(*k).is_some()), + None => false, } - if let Some(v) = obj.get("doubleValue") - && let Some(f) = v.as_float() - { - return Some(Cow::Owned(f.to_string())); +} + +/// Presence semantics for the `body` field: an empty `stringValue` counts as +/// missing, but any other present variant counts as present. +fn log_body_present(value: Option<&Value>) -> bool { + let Some(obj) = value.and_then(Value::as_object) else { + return false; + }; + if let Some(s) = obj.get("stringValue").and_then(Value::as_str) { + return !s.is_empty(); } - None + ["boolValue", "intValue", "doubleValue", "arrayValue", "kvlistValue", "bytesValue"] + .iter() + .any(|k| obj.get(*k).is_some()) } -/// Coerce a plain (non-`AnyValue`) `Value` to a string for matching — -/// used for simple OTLP fields like `severityText`, `traceId`, `spanId`, -/// which are flat strings rather than wrapped `AnyValue` objects. -fn plain_value_to_string(value: &Value) -> Option> { - match value { - Value::Bytes(_) => value.as_str(), - Value::Integer(i) => Some(Cow::Owned(i.to_string())), - Value::Float(f) => Some(Cow::Owned(f.to_string())), - Value::Boolean(b) => Some(Cow::Borrowed(if *b { "true" } else { "false" })), +/// Coerce a plain string `Value` to a non-empty string. Empty strings count +/// as absent, matching the reference adapter's `non_empty` helper. +pub(super) fn non_empty(value: Option<&Value>) -> Option> { + match value?.as_str() { + Some(s) if !s.is_empty() => Some(s), _ => None, } } -/// Linear scan of an OTLP attributes array for the entry whose key -/// matches. -fn find_attribute_value<'a>(attrs: &'a Value, key: &str) -> Option> { - let array = attrs.as_array()?; - for item in array { - if attribute_key(item) == Some(key) { - return item - .as_object() - .and_then(|o| o.get("value")) - .and_then(any_value_to_string); +// ============================================================================= +// Attribute lookup (walks nested kvlistValue). +// ============================================================================= + +pub(super) fn find_attribute_path<'a>( + attrs: Option<&'a Value>, + path: &[String], +) -> Option> { + let array = attrs?.as_array()?; + find_in_kvlist(array, path) +} + +fn find_in_kvlist<'a>(attrs: &'a [Value], path: &[String]) -> Option> { + let first = path.first()?; + for kv in attrs { + if attribute_key(kv) != Some(first.as_str()) { + continue; } + let value = kv.as_object().and_then(|o| o.get("value")); + if path.len() == 1 { + return any_value_string(value); + } + return nested_values(value).and_then(|nested| find_in_kvlist(nested, &path[1..])); } None } -/// Like [`find_attribute_value`] but returns whether the key exists at -/// all — used by `field_exists` so OTLP attributes with non-string -/// `AnyValue` variants (`intValue`, `boolValue`, `arrayValue`, …) still -/// satisfy `exists: true` matchers. -fn attribute_exists(attrs: &Value, key: &str) -> bool { - attrs - .as_array() - .map(|array| array.iter().any(|item| attribute_key(item) == Some(key))) - .unwrap_or(false) +pub(super) fn attribute_exists_path(attrs: Option<&Value>, path: &[String]) -> bool { + let Some(array) = attrs.and_then(Value::as_array) else { + return false; + }; + exists_in_kvlist(array, path) } -fn attribute_index(attrs: &[Value], key: &str) -> Option { - attrs - .iter() - .position(|item| attribute_key(item) == Some(key)) +fn exists_in_kvlist(attrs: &[Value], path: &[String]) -> bool { + let Some(first) = path.first() else { + return false; + }; + for kv in attrs { + if attribute_key(kv) != Some(first.as_str()) { + continue; + } + let value = kv.as_object().and_then(|o| o.get("value")); + if path.len() == 1 { + return any_value_present(value); + } + return nested_values(value) + .map(|nested| exists_in_kvlist(nested, &path[1..])) + .unwrap_or(false); + } + false +} + +/// Unwrap an `AnyValue`'s `kvlistValue.values` array. +fn nested_values(value: Option<&Value>) -> Option<&[Value]> { + value + .and_then(Value::as_object) + .and_then(|o| o.get("kvlistValue")) + .and_then(Value::as_object) + .and_then(|o| o.get("values")) + .and_then(Value::as_array) } fn attribute_key(item: &Value) -> Option<&str> { @@ -423,6 +527,56 @@ fn attribute_key(item: &Value) -> Option<&str> { }) } +// ============================================================================= +// Attribute mutation. +// ============================================================================= + +/// Set or overwrite the first-segment attribute key with a string value. +fn set_string_attr(attrs: &mut Vec, path: &[String], value: &str) { + let Some(key) = path.first() else { + return; + }; + if let Some(entry) = attrs + .iter_mut() + .find(|kv| attribute_key(kv) == Some(key.as_str())) + { + if let Some(obj) = entry.as_object_mut() { + obj.insert("value".into(), make_string_any_value(value)); + } + return; + } + attrs.push(make_attribute_entry(key, make_string_any_value(value))); +} + +/// Remove the first-segment attribute key. Returns whether anything was removed. +fn remove_attr(attrs: &mut Vec, path: &[String]) -> bool { + let Some(key) = path.first() else { + return false; + }; + let before = attrs.len(); + attrs.retain(|kv| attribute_key(kv) != Some(key.as_str())); + attrs.len() < before +} + +/// Remove and return the first-segment attribute entry, preserving its value. +fn remove_attr_kv(attrs: &mut Vec, path: &[String]) -> Option { + let key = path.first()?; + let idx = attrs + .iter() + .position(|kv| attribute_key(kv) == Some(key.as_str()))?; + Some(attrs.remove(idx)) +} + +/// Insert an entry under `key`, removing any existing entry with that key first. +fn upsert_entry(attrs: &mut Vec, key: &str, entry: Value) { + attrs.retain(|kv| attribute_key(kv) != Some(key)); + attrs.push(entry); +} + +// ============================================================================= +// Value construction. +// ============================================================================= + /// Build an `AnyValue` object wrapping a string. fn make_string_any_value(value: &str) -> Value { let mut obj = ObjectMap::new(); @@ -438,52 +592,30 @@ fn make_attribute_entry(key: &str, value: Value) -> Value { Value::Object(obj) } -/// Insert (or overwrite) an attribute entry by key, wrapping the value -/// as a string `AnyValue`. -fn set_attribute(attrs: &mut Vec, key: &str, value: &str) { - let any_value = make_string_any_value(value); - if let Some(idx) = attribute_index(attrs, key) { - if let Some(obj) = attrs[idx].as_object_mut() { - obj.insert("value".into(), any_value); - } - } else { - attrs.push(make_attribute_entry(key, any_value)); +/// Get-or-create the `attributes` array of a record / resource / scope. +fn ensure_attributes(parent: &mut Value) -> Option<&mut Vec> { + let obj = parent.as_object_mut()?; + if !matches!(obj.get("attributes"), Some(Value::Array(_))) { + obj.insert("attributes".into(), Value::Array(Vec::new())); } + obj.get_mut("attributes").and_then(Value::as_array_mut) } -/// Remove an attribute entry by key. Returns whether an entry was -/// removed. -fn remove_attribute(attrs: &mut Vec, key: &str) -> bool { - match attribute_index(attrs, key) { - Some(idx) => { - attrs.remove(idx); - true - } - None => false, - } +/// Borrow the existing `attributes` array, if present. +fn attributes_of_mut(parent: &mut Value) -> Option<&mut Vec> { + parent + .as_object_mut() + .and_then(|o| o.get_mut("attributes")) + .and_then(Value::as_array_mut) } -/// Ensure the log record has an `attributes` array — create an empty one -/// if absent so `set_attribute` has somewhere to push. -fn ensure_attributes_array(log_record: &mut Value) { - let needs_init = log_record - .as_object() - .and_then(|o| o.get("attributes")) - .map(|v| !matches!(v, Value::Array(_))) - .unwrap_or(true); - if needs_init && let Some(obj) = log_record.as_object_mut() { - obj.insert("attributes".into(), Value::Array(Vec::new())); +fn remove_key(record: &mut Value, key: &str) -> bool { + match record.as_object_mut() { + Some(obj) => obj.remove(key).is_some(), + None => false, } } -/// `policy-rs` attribute selectors are multi-segment paths. In OTel mode -/// we collapse them with a literal dot — the OTel convention is that -/// nested attribute keys (e.g. `"service.name"`) live as a single -/// flat key with embedded dots. -fn join_path(segments: &[String]) -> String { - segments.join(".") -} - fn insert_simple_string(record: &mut Value, key: &str, value: &str) { if let Some(obj) = record.as_object_mut() { obj.insert(key.into(), Value::from(value.to_string())); @@ -494,323 +626,217 @@ fn insert_simple_string(record: &mut Value, key: &str, value: &str) { mod tests { use super::*; - /// Build a single OTLP log record `Value` with the given body and a - /// `user_id` string attribute. Keeps the test bodies tiny. - fn record_with_body_and_attr(body: &str, attr_key: &str, attr_val: &str) -> Value { + fn any_int(v: i64) -> Value { + let mut obj = ObjectMap::new(); + obj.insert("intValue".into(), Value::Integer(v)); + Value::Object(obj) + } + + fn record_with_attr(attr_key: &str, attr_val: Value) -> Value { let mut record = ObjectMap::new(); - record.insert("body".into(), make_string_any_value(body)); + record.insert("body".into(), make_string_any_value("hi")); record.insert("severityText".into(), Value::from("INFO".to_string())); record.insert( "attributes".into(), - Value::Array(vec![make_attribute_entry( - attr_key, - make_string_any_value(attr_val), - )]), + Value::Array(vec![make_attribute_entry(attr_key, attr_val)]), ); Value::Object(record) } - fn resource_with_attr(key: &str, val: &str) -> Value { - let mut resource = ObjectMap::new(); - resource.insert( + fn obj_with_attrs(pairs: Vec<(&str, Value)>) -> Value { + let mut o = ObjectMap::new(); + o.insert( "attributes".into(), - Value::Array(vec![make_attribute_entry(key, make_string_any_value(val))]), - ); - Value::Object(resource) - } - - fn scope_with_attr(key: &str, val: &str) -> Value { - let mut scope = ObjectMap::new(); - scope.insert("name".into(), Value::from("test".to_string())); - scope.insert( - "attributes".into(), - Value::Array(vec![make_attribute_entry(key, make_string_any_value(val))]), - ); - Value::Object(scope) - } - - #[test] - fn any_value_string() { - let v = make_string_any_value("hello"); - assert_eq!(any_value_to_string(&v).as_deref(), Some("hello")); - } - - #[test] - fn any_value_int_stringifies() { - let mut obj = ObjectMap::new(); - obj.insert("intValue".into(), Value::Integer(42)); - assert_eq!( - any_value_to_string(&Value::Object(obj)).as_deref(), - Some("42"), - ); - } - - #[test] - fn any_value_bool_stringifies() { - let mut obj = ObjectMap::new(); - obj.insert("boolValue".into(), Value::Boolean(true)); - assert_eq!( - any_value_to_string(&Value::Object(obj)).as_deref(), - Some("true"), + Value::Array( + pairs + .into_iter() + .map(|(k, v)| make_attribute_entry(k, v)) + .collect(), + ), ); + Value::Object(o) } - #[test] - fn any_value_double_stringifies() { - let mut obj = ObjectMap::new(); - obj.insert( - "doubleValue".into(), - Value::Float(ordered_float::NotNan::new(1.5).unwrap()), - ); - assert!(matches!( - any_value_to_string(&Value::Object(obj)).as_deref(), - Some(s) if s.starts_with("1.5"), - )); - } - - #[test] - fn any_value_array_returns_none_for_matching() { - let mut obj = ObjectMap::new(); - obj.insert("arrayValue".into(), Value::Array(vec![])); - assert_eq!(any_value_to_string(&Value::Object(obj)), None); + fn get(adapter: &OtlpLogAdapter, sel: LogFieldSelector) -> Option { + adapter.get_field(&sel).map(|c| c.into_owned()) } #[test] - fn get_body_string() { - let mut record = record_with_body_and_attr("hi", "user_id", "42"); + fn matches_string_attribute() { + let mut record = record_with_attr("user_id", make_string_any_value("42")); let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); assert_eq!( - adapter - .get_field(&LogFieldSelector::Simple(LogField::Body)) - .as_deref(), - Some("hi"), + get( + &adapter, + LogFieldSelector::LogAttribute(vec!["user_id".into()]) + ), + Some("42".to_string()) ); } #[test] - fn get_log_attribute_flat() { - let mut record = record_with_body_and_attr("hi", "user_id", "42"); + fn int_attribute_is_not_matchable_but_exists() { + let mut record = record_with_attr("count", any_int(42)); let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + // get_field must return None (not "42") so a regex redact won't fire. assert_eq!( - adapter - .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) - .as_deref(), - Some("42"), + get(&adapter, LogFieldSelector::LogAttribute(vec!["count".into()])), + None ); + // but exists: true must still match. + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["count".into()]))); } #[test] - fn get_log_attribute_nested_dot_joins() { - let mut record = record_with_body_and_attr("x", "http.method", "GET"); + fn empty_string_simple_field_is_absent() { + let mut record = ObjectMap::new(); + record.insert("spanId".into(), Value::from(String::new())); + record.insert("severityText".into(), Value::from("INFO".to_string())); + let mut record = Value::Object(record); let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - assert_eq!( - adapter - .get_field(&LogFieldSelector::LogAttribute(vec![ - "http".to_string(), - "method".to_string() - ])) - .as_deref(), - Some("GET"), - ); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::SpanId))); + assert!(adapter.field_exists(&LogFieldSelector::Simple(LogField::SeverityText))); } #[test] - fn get_resource_attribute_via_immutable_ref() { - let mut record = record_with_body_and_attr("x", "noise", "v"); - let resource = resource_with_attr("service.name", "frontend"); - let adapter = OtlpLogAdapter::new(&mut record, Some(&resource), None, None, None); - assert_eq!( - adapter - .get_field(&LogFieldSelector::ResourceAttribute(vec![ - "service.name".to_string() - ])) - .as_deref(), - Some("frontend"), - ); - } - - #[test] - fn get_scope_attribute_via_immutable_ref() { - let mut record = record_with_body_and_attr("x", "noise", "v"); - let scope = scope_with_attr("library", "tracer"); - let adapter = OtlpLogAdapter::new(&mut record, None, Some(&scope), None, None); + fn hex_span_id_is_matched_verbatim() { + let mut record = ObjectMap::new(); + record.insert("spanId".into(), Value::from("7370616e30303031".to_string())); + let mut record = Value::Object(record); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); assert_eq!( - adapter - .get_field(&LogFieldSelector::ScopeAttribute(vec![ - "library".to_string() - ])) - .as_deref(), - Some("tracer"), + get(&adapter, LogFieldSelector::Simple(LogField::SpanId)), + Some("7370616e30303031".to_string()) ); } #[test] - fn get_resource_schema_url() { - let mut record = record_with_body_and_attr("x", "noise", "v"); - let schema = Value::from("https://opentelemetry.io/schemas/1.0".to_string()); - let adapter = OtlpLogAdapter::new(&mut record, None, None, Some(&schema), None); - assert_eq!( - adapter - .get_field(&LogFieldSelector::Simple(LogField::ResourceSchemaUrl)) - .as_deref(), - Some("https://opentelemetry.io/schemas/1.0"), - ); - } + fn body_present_only_when_non_empty() { + // Missing body. + let mut record = Value::Object(ObjectMap::new()); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); - #[test] - fn field_exists_for_non_string_attribute() { - // intValue attribute must satisfy exists matchers even though - // get_field returns Some("42"). + // Empty AnyValue (body: {}) counts as missing. let mut record = ObjectMap::new(); - record.insert("body".into(), make_string_any_value("x")); - let mut attr_value = ObjectMap::new(); - attr_value.insert("intValue".into(), Value::Integer(42)); - record.insert( - "attributes".into(), - Value::Array(vec![make_attribute_entry( - "count", - Value::Object(attr_value), - )]), - ); + record.insert("body".into(), Value::Object(ObjectMap::new())); let mut record = Value::Object(record); let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["count".to_string()]))); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); } #[test] - fn set_attribute_overwrites_existing() { - let mut record = record_with_body_and_attr("x", "user_id", "42"); + fn add_body_when_missing() { + let mut record = Value::Object(ObjectMap::new()); { let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.set_field( - &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), - "99", - ); + adapter.set_field(&LogFieldSelector::Simple(LogField::Body), "[no body provided]"); } let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); assert_eq!( - adapter - .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) - .as_deref(), - Some("99"), + get(&adapter, LogFieldSelector::Simple(LogField::Body)), + Some("[no body provided]".to_string()) ); } #[test] - fn set_attribute_creates_when_absent() { - let mut record = ObjectMap::new(); - record.insert("body".into(), make_string_any_value("x")); - let mut record = Value::Object(record); - { - let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.set_field( - &LogFieldSelector::LogAttribute(vec!["new".to_string()]), - "v", - ); - } + fn walks_nested_kvlist() { + let mut inner = ObjectMap::new(); + inner.insert( + "values".into(), + Value::Array(vec![make_attribute_entry( + "method", + make_string_any_value("GET"), + )]), + ); + let mut http_val = ObjectMap::new(); + http_val.insert("kvlistValue".into(), Value::Object(inner)); + let mut record = record_with_attr("http", Value::Object(http_val)); let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); assert_eq!( - adapter - .get_field(&LogFieldSelector::LogAttribute(vec!["new".to_string()])) - .as_deref(), - Some("v"), + get( + &adapter, + LogFieldSelector::LogAttribute(vec!["http".into(), "method".into()]) + ), + Some("GET".to_string()) ); + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec![ + "http".into(), + "method".into() + ]))); } #[test] - fn set_simple_body_replaces_any_value() { - let mut record = record_with_body_and_attr("old", "k", "v"); + fn resource_attribute_is_mutable() { + let mut record = record_with_attr("noise", make_string_any_value("v")); + let mut resource = obj_with_attrs(vec![("existing", make_string_any_value("x"))]); { - let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.set_field(&LogFieldSelector::Simple(LogField::Body), "new"); + let mut adapter = + OtlpLogAdapter::new(&mut record, Some(&mut resource), None, None, None); + adapter.set_field( + &LogFieldSelector::ResourceAttribute(vec!["processed_by".into()]), + "policy", + ); } - let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + let adapter = OtlpLogAdapter::new(&mut record, Some(&mut resource), None, None, None); assert_eq!( - adapter - .get_field(&LogFieldSelector::Simple(LogField::Body)) - .as_deref(), - Some("new"), + get( + &adapter, + LogFieldSelector::ResourceAttribute(vec!["processed_by".into()]) + ), + Some("policy".to_string()) ); } #[test] - fn set_resource_attribute_is_noop() { - let mut record = record_with_body_and_attr("x", "k", "v"); - let resource = resource_with_attr("a", "b"); - let resource_before = resource.clone(); + fn scope_attribute_remove_and_rename() { + let mut record = record_with_attr("noise", make_string_any_value("v")); + let mut scope = obj_with_attrs(vec![ + ("secret", make_string_any_value("abc")), + ("old", make_string_any_value("val")), + ]); { - let mut adapter = OtlpLogAdapter::new(&mut record, Some(&resource), None, None, None); - adapter.set_field( - &LogFieldSelector::ResourceAttribute(vec!["a".to_string()]), - "ignored", + let mut adapter = + OtlpLogAdapter::new(&mut record, None, Some(&mut scope), None, None); + assert!(adapter.delete_field(&LogFieldSelector::ScopeAttribute(vec!["secret".into()]))); + adapter.move_field( + &LogFieldSelector::ScopeAttribute(vec!["old".into()]), + &LogFieldSelector::ScopeAttribute(vec!["new".into()]), ); } - assert_eq!(resource, resource_before, "resource must not be mutated"); - } - - #[test] - fn delete_attribute_present() { - let mut record = record_with_body_and_attr("x", "user_id", "42"); - let removed = { - let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) - }; - assert!(removed); - let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - assert!( - adapter - .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) - .is_none(), - ); - } - - #[test] - fn delete_attribute_absent_returns_false() { - let mut record = record_with_body_and_attr("x", "user_id", "42"); - let removed = { - let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["other".to_string()])) - }; - assert!(!removed); - } - - #[test] - fn delete_simple_body() { - let mut record = record_with_body_and_attr("x", "k", "v"); - let removed = { - let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.delete_field(&LogFieldSelector::Simple(LogField::Body)) - }; - assert!(removed); - let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - assert!( - adapter - .get_field(&LogFieldSelector::Simple(LogField::Body)) - .is_none(), + let adapter = OtlpLogAdapter::new(&mut record, None, Some(&mut scope), None, None); + assert!(get(&adapter, LogFieldSelector::ScopeAttribute(vec!["secret".into()])).is_none()); + assert!(get(&adapter, LogFieldSelector::ScopeAttribute(vec!["old".into()])).is_none()); + assert_eq!( + get(&adapter, LogFieldSelector::ScopeAttribute(vec!["new".into()])), + Some("val".to_string()) ); } #[test] - fn move_attribute_within_log_attributes() { - let mut record = record_with_body_and_attr("x", "usr", "admin"); + fn redact_regex_skips_non_string_value() { + // Mirrors logs_transform_redact_regex_non_string_value: an intValue + // attribute must be left untouched because get_field returns None. + let mut record = record_with_attr("count", any_int(42)); { - let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.move_field( - &LogFieldSelector::LogAttribute(vec!["usr".to_string()]), - &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + // The engine reads the value first; None means redact is skipped. + assert!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["count".into()])) + .is_none() ); } - let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - assert!( - adapter - .get_field(&LogFieldSelector::LogAttribute(vec!["usr".to_string()])) - .is_none(), - ); - assert_eq!( - adapter - .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) - .as_deref(), - Some("admin"), - ); + // The underlying intValue is still present and unchanged. + let count = record + .get("attributes") + .and_then(Value::as_array) + .and_then(|a| a.first()) + .and_then(|kv| kv.as_object()) + .and_then(|o| o.get("value")) + .and_then(Value::as_object) + .and_then(|o| o.get("intValue")) + .cloned(); + assert_eq!(count, Some(Value::Integer(42))); } } diff --git a/src/transforms/policy/otlp_metric_adapter.rs b/src/transforms/policy/otlp_metric_adapter.rs new file mode 100644 index 0000000000000..d1a464d6f84b4 --- /dev/null +++ b/src/transforms/policy/otlp_metric_adapter.rs @@ -0,0 +1,217 @@ +//! OTLP metrics adapter for the `policy` transform (`mode: otel`). +//! +//! When Vector's `opentelemetry` source decodes OTLP metrics with +//! `use_otlp_decoding`, the metrics arrive as a `Log` event shaped like +//! `{ resourceMetrics: [...] }`. This module iterates every metric inside the +//! envelope, evaluates it through `policy-rs` (match-only — metrics are not +//! transformed), drops metrics whose winning policy says so, and prunes empty +//! `scopeMetrics` / `resourceMetrics`. +//! +//! Mirrors the conformance reference (`MetricContext` in +//! `policy-conformance/runners/rs/src/eval.rs`): a metric is matched on the +//! attributes of its **first** data point, its descriptor fields, its type +//! (gauge/sum/…) and aggregation temporality. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::MetricField; +use policy_rs::{ + EvaluateResult, Matchable, MetricFieldSelector, PolicyEngine, PolicySnapshot, + engine::MetricSignal, +}; +use vector_lib::event::{LogEvent, Value}; + +use super::internal_events::{DropReason, emit_dropped}; +use super::otlp_adapter::{attribute_exists_path, find_attribute_path, non_empty}; + +/// Data-variant keys of an OTLP `Metric` (proto3 JSON), paired with the +/// `MetricFieldSelector::Type` string the engine matches against. +const METRIC_TYPES: [(&str, &str); 5] = [ + ("gauge", "METRIC_TYPE_GAUGE"), + ("sum", "METRIC_TYPE_SUM"), + ("histogram", "METRIC_TYPE_HISTOGRAM"), + ("exponentialHistogram", "METRIC_TYPE_EXPONENTIAL_HISTOGRAM"), + ("summary", "METRIC_TYPE_SUMMARY"), +]; + +/// Iterate every metric in an OTLP metrics envelope, dropping filtered ones. +/// Returns `Some(log)` to forward or `None` to drop the whole event. +pub(super) async fn evaluate_metrics_envelope( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + mut log: LogEvent, +) -> Option { + let Some(resource_metrics) = log.get_mut("resourceMetrics").and_then(Value::as_array_mut) + else { + return Some(log); + }; + + let mut i = 0; + while i < resource_metrics.len() { + // Metrics are read-only, so immutable snapshots of resource/scope are + // enough (no remove/re-insert dance needed). + let resource = resource_metrics[i].get("resource").cloned(); + let resource_schema_url = resource_metrics[i].get("schemaUrl").cloned(); + + let mut prune_rm = false; + + if let Some(scope_metrics) = resource_metrics[i] + .get_mut("scopeMetrics") + .and_then(Value::as_array_mut) + { + let mut j = 0; + while j < scope_metrics.len() { + let scope = scope_metrics[j].get("scope").cloned(); + let scope_schema_url = scope_metrics[j].get("schemaUrl").cloned(); + + let mut prune_sm = false; + + if let Some(metrics) = scope_metrics[j] + .get_mut("metrics") + .and_then(Value::as_array_mut) + { + // Evaluate first (immutable borrows), then retain by index. + let mut keep = Vec::with_capacity(metrics.len()); + for metric in metrics.iter() { + let adapter = MetricAdapter { + metric, + resource: resource.as_ref(), + scope: scope.as_ref(), + resource_schema_url: resource_schema_url.as_ref(), + scope_schema_url: scope_schema_url.as_ref(), + }; + let drop = matches!( + engine.evaluate(snapshot, &adapter).await, + Ok(EvaluateResult::Drop { .. }) + ); + if drop { + emit_dropped(DropReason::PolicyDrop); + } + keep.push(!drop); + } + let mut idx = 0; + metrics.retain(|_| { + let k = keep[idx]; + idx += 1; + k + }); + prune_sm = metrics.is_empty(); + } + + if prune_sm { + scope_metrics.remove(j); + } else { + j += 1; + } + } + prune_rm = scope_metrics.is_empty(); + } + + if prune_rm { + resource_metrics.remove(i); + } else { + i += 1; + } + } + + if resource_metrics.is_empty() { + None + } else { + Some(log) + } +} + +/// Adapter exposing a single OTLP `metric` (plus parent resource/scope) to the +/// `policy-rs` engine. Read-only: metrics are filtered, never transformed. +struct MetricAdapter<'a> { + metric: &'a Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, +} + +impl MetricAdapter<'_> { + /// The data variant object (gauge/sum/…) and its canonical type string. + fn data(&self) -> Option<(&Value, &'static str)> { + let obj = self.metric.as_object()?; + METRIC_TYPES + .iter() + .find_map(|(key, ty)| obj.get(*key).map(|v| (v, *ty))) + } + + /// Attributes array of the first data point, if any. + fn first_datapoint_attributes(&self) -> Option<&Value> { + let (data, _) = self.data()?; + data.as_object() + .and_then(|o| o.get("dataPoints")) + .and_then(Value::as_array) + .and_then(|points| points.first()) + .and_then(|p| p.as_object()) + .and_then(|o| o.get("attributes")) + } + + fn resource_attributes(&self) -> Option<&Value> { + self.resource.and_then(|r| r.get("attributes")) + } + + fn scope_attributes(&self) -> Option<&Value> { + self.scope.and_then(|s| s.get("attributes")) + } +} + +impl Matchable for MetricAdapter<'_> { + type Signal = MetricSignal; + + fn get_field(&self, field: &MetricFieldSelector) -> Option> { + match field { + MetricFieldSelector::Simple(MetricField::Name) => non_empty(self.metric.get("name")), + MetricFieldSelector::Simple(MetricField::Description) => { + non_empty(self.metric.get("description")) + } + MetricFieldSelector::Simple(MetricField::Unit) => non_empty(self.metric.get("unit")), + MetricFieldSelector::Simple(MetricField::ScopeName) => { + non_empty(self.scope.and_then(|s| s.get("name"))) + } + MetricFieldSelector::Simple(MetricField::ScopeVersion) => { + non_empty(self.scope.and_then(|s| s.get("version"))) + } + MetricFieldSelector::Simple(MetricField::ResourceSchemaUrl) => { + non_empty(self.resource_schema_url) + } + MetricFieldSelector::Simple(MetricField::ScopeSchemaUrl) => { + non_empty(self.scope_schema_url) + } + MetricFieldSelector::Simple(MetricField::Unspecified) => None, + MetricFieldSelector::DatapointAttribute(path) => { + find_attribute_path(self.first_datapoint_attributes(), path) + } + MetricFieldSelector::ResourceAttribute(path) => { + find_attribute_path(self.resource_attributes(), path) + } + MetricFieldSelector::ScopeAttribute(path) => { + find_attribute_path(self.scope_attributes(), path) + } + MetricFieldSelector::Type => self.data().map(|(_, ty)| Cow::Borrowed(ty)), + MetricFieldSelector::Temporality => { + let (data, _) = self.data()?; + non_empty(data.as_object().and_then(|o| o.get("aggregationTemporality"))) + } + } + } + + fn field_exists(&self, field: &MetricFieldSelector) -> bool { + match field { + MetricFieldSelector::DatapointAttribute(path) => { + attribute_exists_path(self.first_datapoint_attributes(), path) + } + MetricFieldSelector::ResourceAttribute(path) => { + attribute_exists_path(self.resource_attributes(), path) + } + MetricFieldSelector::ScopeAttribute(path) => { + attribute_exists_path(self.scope_attributes(), path) + } + _ => self.get_field(field).is_some(), + } + } +} diff --git a/src/transforms/policy/otlp_trace_adapter.rs b/src/transforms/policy/otlp_trace_adapter.rs new file mode 100644 index 0000000000000..ae1e295f94af8 --- /dev/null +++ b/src/transforms/policy/otlp_trace_adapter.rs @@ -0,0 +1,307 @@ +//! OTLP traces adapter for the `policy` transform (`mode: otel`). +//! +//! Vector's `opentelemetry` source decodes OTLP traces into a `Trace` event +//! shaped like `{ resourceSpans: [...] }`. This module iterates every span, +//! evaluates it through `policy-rs` (`evaluate_trace`, which applies OTel +//! consistent-probability sampling), drops filtered spans, writes the sampling +//! threshold back into the span's `traceState`, and prunes empty +//! `scopeSpans` / `resourceSpans`. +//! +//! Mirrors the conformance reference (`MutTraceContext` in +//! `policy-conformance/runners/rs/src/eval.rs`). Resource and scope are +//! read-only in trace mode; only the span (its `traceState`) is mutated. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::TraceField; +use policy_rs::{ + EvaluateResult, Matchable, PolicyEngine, PolicySnapshot, TraceFieldSelector, Transformable, + engine::TraceSignal, +}; +use vector_lib::event::{TraceEvent, Value}; + +use super::internal_events::{DropReason, emit_dropped}; +use super::otlp_adapter::{attribute_exists_path, find_attribute_path, non_empty}; + +/// Iterate every span in an OTLP traces envelope, sampling/dropping in place. +/// Returns `true` if any span survives (forward the event), `false` if the +/// envelope is now empty and the event should be dropped. +pub(super) async fn evaluate_traces_envelope( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + trace: &mut TraceEvent, +) -> bool { + let Some(resource_spans) = trace + .value_mut() + .get_mut("resourceSpans") + .and_then(Value::as_array_mut) + else { + // Not an envelope. Forward unchanged. + return true; + }; + + let mut i = 0; + while i < resource_spans.len() { + let resource = resource_spans[i].get("resource").cloned(); + let resource_schema_url = resource_spans[i].get("schemaUrl").cloned(); + + let mut prune_rs = false; + + if let Some(scope_spans) = resource_spans[i] + .get_mut("scopeSpans") + .and_then(Value::as_array_mut) + { + let mut j = 0; + while j < scope_spans.len() { + let scope = scope_spans[j].get("scope").cloned(); + let scope_schema_url = scope_spans[j].get("schemaUrl").cloned(); + + let mut prune_ss = false; + + if let Some(spans) = scope_spans[j] + .get_mut("spans") + .and_then(Value::as_array_mut) + { + let mut k = 0; + while k < spans.len() { + let result = { + let mut adapter = TraceAdapter { + span: &mut spans[k], + resource: resource.as_ref(), + scope: scope.as_ref(), + resource_schema_url: resource_schema_url.as_ref(), + scope_schema_url: scope_schema_url.as_ref(), + }; + engine.evaluate_trace(snapshot, &mut adapter).await + }; + + let keep = match result { + Ok(EvaluateResult::Drop { .. }) => { + emit_dropped(DropReason::PolicyDrop); + false + } + Ok(EvaluateResult::Sample { keep: false, .. }) => { + emit_dropped(DropReason::SampleRejected); + false + } + Ok(_) => true, + Err(error) => { + error!( + message = "Policy evaluation failed; OTLP span passed through unchanged.", + %error, + ); + true + } + }; + + if keep { + k += 1; + } else { + spans.remove(k); + } + } + prune_ss = spans.is_empty(); + } + + if prune_ss { + scope_spans.remove(j); + } else { + j += 1; + } + } + prune_rs = scope_spans.is_empty(); + } + + if prune_rs { + resource_spans.remove(i); + } else { + i += 1; + } + } + + !resource_spans.is_empty() +} + +/// Adapter exposing a single OTLP span (plus parent resource/scope) to the +/// `policy-rs` engine. The span is mutable so the engine can write the +/// sampling threshold into `traceState`; resource/scope are read-only. +struct TraceAdapter<'a> { + span: &'a mut Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, +} + +impl TraceAdapter<'_> { + fn span_attributes(&self) -> Option<&Value> { + self.span.get("attributes") + } + + fn resource_attributes(&self) -> Option<&Value> { + self.resource.and_then(|r| r.get("attributes")) + } + + fn scope_attributes(&self) -> Option<&Value> { + self.scope.and_then(|s| s.get("attributes")) + } + + /// First non-empty span event name, used by `EventName` matchers. + fn event_name(&self) -> Option> { + let events = self.span.get("events").and_then(Value::as_array)?; + for event in events { + if let Some(name) = non_empty(event.as_object().and_then(|o| o.get("name"))) { + return Some(name); + } + } + None + } + + /// Map the OTel `Status.code` enum to the policy `SPAN_STATUS_CODE_*` form. + /// + /// A span carries a status only if the `status` message is present. Its + /// `code` defaults to `STATUS_CODE_UNSET` (0), which proto3 omits on the + /// wire — so a present-but-empty `status` (or one whose `code` decoded to + /// the default) means UNSET, not "no status". A truly absent `status` + /// message yields `None`. + fn span_status(&self) -> Option> { + let status = self.span.get("status").and_then(Value::as_object)?; + match status.get("code").and_then(Value::as_str).as_deref() { + Some("STATUS_CODE_OK") => Some(Cow::Borrowed("SPAN_STATUS_CODE_OK")), + Some("STATUS_CODE_ERROR") => Some(Cow::Borrowed("SPAN_STATUS_CODE_ERROR")), + Some("STATUS_CODE_UNSET") | None => { + Some(Cow::Borrowed("SPAN_STATUS_CODE_UNSPECIFIED")) + } + Some(_) => None, + } + } +} + +impl Matchable for TraceAdapter<'_> { + type Signal = TraceSignal; + + fn get_field(&self, field: &TraceFieldSelector) -> Option> { + match field { + TraceFieldSelector::Simple(TraceField::Name) => non_empty(self.span.get("name")), + TraceFieldSelector::Simple(TraceField::TraceId) => non_empty(self.span.get("traceId")), + TraceFieldSelector::Simple(TraceField::SpanId) => non_empty(self.span.get("spanId")), + TraceFieldSelector::Simple(TraceField::ParentSpanId) => { + non_empty(self.span.get("parentSpanId")) + } + TraceFieldSelector::Simple(TraceField::TraceState) => { + non_empty(self.span.get("traceState")) + } + TraceFieldSelector::Simple(TraceField::ScopeName) => { + non_empty(self.scope.and_then(|s| s.get("name"))) + } + TraceFieldSelector::Simple(TraceField::ScopeVersion) => { + non_empty(self.scope.and_then(|s| s.get("version"))) + } + TraceFieldSelector::Simple(TraceField::ResourceSchemaUrl) => { + non_empty(self.resource_schema_url) + } + TraceFieldSelector::Simple(TraceField::ScopeSchemaUrl) => { + non_empty(self.scope_schema_url) + } + TraceFieldSelector::Simple(TraceField::Unspecified) => None, + TraceFieldSelector::SpanAttribute(path) => { + find_attribute_path(self.span_attributes(), path) + } + TraceFieldSelector::ResourceAttribute(path) => { + find_attribute_path(self.resource_attributes(), path) + } + TraceFieldSelector::ScopeAttribute(path) => { + find_attribute_path(self.scope_attributes(), path) + } + TraceFieldSelector::SpanKind => non_empty(self.span.get("kind")), + TraceFieldSelector::SpanStatus => self.span_status(), + TraceFieldSelector::EventName => self.event_name(), + // Not exercised by the conformance suite / not representable here. + TraceFieldSelector::EventAttribute(_) + | TraceFieldSelector::LinkTraceId + | TraceFieldSelector::SamplingThreshold => None, + } + } + + fn field_exists(&self, field: &TraceFieldSelector) -> bool { + match field { + TraceFieldSelector::SpanAttribute(path) => { + attribute_exists_path(self.span_attributes(), path) + } + TraceFieldSelector::ResourceAttribute(path) => { + attribute_exists_path(self.resource_attributes(), path) + } + TraceFieldSelector::ScopeAttribute(path) => { + attribute_exists_path(self.scope_attributes(), path) + } + _ => self.get_field(field).is_some(), + } + } +} + +impl Transformable for TraceAdapter<'_> { + fn set_field(&mut self, field: &TraceFieldSelector, value: &str) { + if matches!(field, TraceFieldSelector::SamplingThreshold) { + let current = self + .span + .get("traceState") + .and_then(Value::as_str) + .unwrap_or(Cow::Borrowed("")); + let merged = merge_ot_tracestate(¤t, &format!("th:{value}")); + if let Some(obj) = self.span.as_object_mut() { + obj.insert("traceState".into(), Value::from(merged)); + } + } + // Other trace transforms are not exercised by the conformance suite. + } + + fn delete_field(&mut self, _field: &TraceFieldSelector) -> bool { + false + } + + fn move_field(&mut self, _from: &TraceFieldSelector, _to: &TraceFieldSelector) {} +} + +/// Merge an OpenTelemetry sub-key (e.g. `"th:8000"`) into a W3C tracestate +/// string under the `ot` vendor key, replacing any existing entry with the +/// same sub-key. Ported verbatim from the conformance reference adapter. +fn merge_ot_tracestate(tracestate: &str, sub_kv: &str) -> String { + let sub_key = sub_kv.split(':').next().unwrap_or(sub_kv); + + let mut ot_parts: Vec<&str> = Vec::new(); + let mut other_vendors: Vec<&str> = Vec::new(); + + if !tracestate.is_empty() { + for vendor in tracestate.split(',') { + let vendor = vendor.trim(); + if vendor.is_empty() { + continue; + } + if let Some(ot_value) = vendor.strip_prefix("ot=") { + for part in ot_value.split(';') { + let part = part.trim(); + if part.is_empty() { + continue; + } + let part_key = part.split(':').next().unwrap_or(part); + if part_key != sub_key { + ot_parts.push(part); + } + } + } else { + other_vendors.push(vendor); + } + } + } + + let mut result = format!("ot={}", ot_parts.join(";")); + if !ot_parts.is_empty() { + result.push(';'); + } + result.push_str(sub_kv); + if !other_vendors.is_empty() { + result.push(','); + result.push_str(&other_vendors.join(",")); + } + result +} diff --git a/src/transforms/policy/transform.rs b/src/transforms/policy/transform.rs index 5abf079548d4a..109b70b1ba615 100644 --- a/src/transforms/policy/transform.rs +++ b/src/transforms/policy/transform.rs @@ -15,6 +15,8 @@ use super::config::PolicyMode; use super::field_mapping::FieldMapping; use super::internal_events::{DropReason, emit_dropped}; use super::otlp_adapter::evaluate_envelope; +use super::otlp_metric_adapter::evaluate_metrics_envelope; +use super::otlp_trace_adapter::evaluate_traces_envelope; /// Per-task state for the `policy` transform. /// @@ -60,16 +62,23 @@ impl TaskTransform for Policy { Box::pin(stream! { while let Some(event) = input_rx.next().await { + // Pull a fresh snapshot every event so live-reloaded policies + // take effect on the next pass. Snapshots are cheap (Arc clone + // of an immutable inner). + let snapshot = registry.snapshot(); match event { Event::Log(log) => { - // Pull a fresh snapshot every event so live-reloaded - // policies take effect on the next pass. Snapshots - // are cheap (Arc clone of an immutable inner). - let snapshot = registry.snapshot(); let outcome = match mode { PolicyMode::Flat => { evaluate_flat(&engine, &snapshot, &mapping, log).await } + // In OTel mode a `Log` event is either a logs + // envelope (`resourceLogs`) or a metrics envelope + // (`resourceMetrics`) — the `opentelemetry` source + // decodes both into `Log` events. + PolicyMode::Otel if log.contains("resourceMetrics") => { + evaluate_metrics_envelope(&engine, &snapshot, log).await + } PolicyMode::Otel => { evaluate_envelope(&engine, &snapshot, log).await } @@ -78,10 +87,22 @@ impl TaskTransform for Policy { yield Event::Log(forwarded); } } + Event::Trace(mut trace) => { + // OTLP traces arrive as `Trace` events + // (`resourceSpans`). Flat mode has no trace mapping, so + // only OTel mode evaluates them; otherwise pass through. + let keep = match mode { + PolicyMode::Otel => { + evaluate_traces_envelope(&engine, &snapshot, &mut trace).await + } + PolicyMode::Flat => true, + }; + if keep { + yield Event::Trace(trace); + } + } + // Native Vector metrics (not OTLP envelopes) pass through. other => { - // policy-rs only targets the log signal in this - // integration. Metrics and traces are forwarded - // untouched regardless of mode. yield other; } } From ecf881225b62388eca77ab84f75ed53e34312aef Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Thu, 28 May 2026 15:43:40 -0400 Subject: [PATCH 5/9] revert protos for now --- lib/codecs/src/decoding/format/otlp.rs | 6 - lib/opentelemetry-proto/src/common.rs | 3 - .../collector/logs/v1/logs_service.proto | 2 + .../metrics/v1/metrics_service.proto | 2 + .../collector/trace/v1/trace_service.proto | 2 + .../proto/common/v1/common.proto | 79 +---------- .../opentelemetry/proto/logs/v1/logs.proto | 33 +---- .../proto/metrics/v1/metrics.proto | 103 +++----------- .../proto/resource/v1/resource.proto | 10 +- .../opentelemetry/proto/trace/v1/trace.proto | 127 +++--------------- src/sources/opentelemetry/tests.rs | 49 ------- 11 files changed, 59 insertions(+), 357 deletions(-) diff --git a/lib/codecs/src/decoding/format/otlp.rs b/lib/codecs/src/decoding/format/otlp.rs index 94ec875b2f3e6..af35d2d2c76e1 100644 --- a/lib/codecs/src/decoding/format/otlp.rs +++ b/lib/codecs/src/decoding/format/otlp.rs @@ -246,7 +246,6 @@ mod tests { resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, - entity_refs: vec![], }), scope_logs: vec![ScopeLogs { scope: None, @@ -261,7 +260,6 @@ mod tests { trace_id: vec![], span_id: vec![], observed_time_unix_nano: 0, - event_name: String::new(), }], schema_url: String::new(), }], @@ -278,7 +276,6 @@ mod tests { resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, - entity_refs: vec![], }), scope_metrics: vec![ScopeMetrics { scope: None, @@ -286,7 +283,6 @@ mod tests { name: "test_metric".to_string(), description: String::new(), unit: String::new(), - metadata: vec![], data: None, }], schema_url: String::new(), @@ -304,7 +300,6 @@ mod tests { resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, - entity_refs: vec![], }), scope_spans: vec![ScopeSpans { scope: None, @@ -324,7 +319,6 @@ mod tests { links: vec![], dropped_links_count: 0, status: None, - flags: 0, }], schema_url: String::new(), }], diff --git a/lib/opentelemetry-proto/src/common.rs b/lib/opentelemetry-proto/src/common.rs index e41f90eb7a7b8..d2cb2876447dc 100644 --- a/lib/opentelemetry-proto/src/common.rs +++ b/lib/opentelemetry-proto/src/common.rs @@ -20,9 +20,6 @@ impl From for Value { .collect::>(), ), PBValue::KvlistValue(arr) => kv_list_into_value(arr.values), - // String-table reference (profiles signal). We don't carry the - // referenced string table here, so there's nothing to resolve. - PBValue::StringValueStrindex(_) => Value::Null, } } } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto index 8be5cf75ef177..8260d8aaeb821 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto @@ -28,6 +28,8 @@ option go_package = "go.opentelemetry.io/proto/otlp/collector/logs/v1"; // OpenTelemetry and an collector, or between an collector and a central collector (in this // case logs are sent/received to/from multiple Applications). service LogsService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. rpc Export(ExportLogsServiceRequest) returns (ExportLogsServiceResponse) {} } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto index bc0242844151b..dd48f1ad3a168 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto @@ -28,6 +28,8 @@ option go_package = "go.opentelemetry.io/proto/otlp/collector/metrics/v1"; // instrumented with OpenTelemetry and a collector, or between a collector and a // central collector. service MetricsService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. rpc Export(ExportMetricsServiceRequest) returns (ExportMetricsServiceResponse) {} } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto index efbbedbe4545a..d6fe67f9e553d 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto @@ -28,6 +28,8 @@ option go_package = "go.opentelemetry.io/proto/otlp/collector/trace/v1"; // OpenTelemetry and a collector, or between a collector and a central collector (in this // case spans are sent/received to/from multiple Applications). service TraceService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. rpc Export(ExportTraceServiceRequest) returns (ExportTraceServiceResponse) {} } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto index 58a911ad49851..f7ee8f2653cc4 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto @@ -22,7 +22,7 @@ option java_package = "io.opentelemetry.proto.common.v1"; option java_outer_classname = "CommonProto"; option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; -// Represents any type of attribute value. AnyValue may contain a +// AnyValue is used to represent any type of attribute value. AnyValue may contain a // primitive value such as a string or integer or it may contain an arbitrary nested // object containing arrays, key-value lists and primitives. message AnyValue { @@ -36,17 +36,6 @@ message AnyValue { ArrayValue array_value = 5; KeyValueList kvlist_value = 6; bytes bytes_value = 7; - // Reference to the string value in ProfilesDictionary.string_table. - // - // Note: This is currently used exclusively in the Profiling signal. - // Implementers of OTLP receivers for signals other than Profiling should - // treat the presence of this value as a non-fatal issue. - // Log an error or warning indicating an unexpected field intended for the - // Profiling signal and process the data as if this value were absent or - // empty, ignoring its semantic content for the non-Profiling signal. - // - // Status: [Development] - int32 string_value_strindex = 8; } } @@ -65,90 +54,28 @@ message ArrayValue { message KeyValueList { // A collection of key/value pairs of key-value pairs. The list may be empty (may // contain 0 elements). - // // The keys MUST be unique (it is not allowed to have more than one // value with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated KeyValue values = 1; } -// Represents a key-value pair that is used to store Span attributes, Link +// KeyValue is a key-value pair that is used to store Span attributes, Link // attributes, etc. message KeyValue { - // The key name of the pair. - // key_ref MUST NOT be set if key is used. string key = 1; - - // The value of the pair. AnyValue value = 2; - - // Reference to the string key in ProfilesDictionary.string_table. - // key MUST NOT be set if key_strindex is used. - // - // Note: This is currently used exclusively in the Profiling signal. - // Implementers of OTLP receivers for signals other than Profiling should - // treat the presence of this key as a non-fatal issue. - // Log an error or warning indicating an unexpected field intended for the - // Profiling signal and process the data as if this value were absent or - // empty, ignoring its semantic content for the non-Profiling signal. - // - // Status: [Development] - int32 key_strindex = 3; } // InstrumentationScope is a message representing the instrumentation scope information -// such as the fully qualified name and version. +// such as the fully qualified name and version. message InstrumentationScope { - // A name denoting the Instrumentation scope. // An empty instrumentation scope name means the name is unknown. string name = 1; - - // Defines the version of the instrumentation scope. - // An empty instrumentation scope version means the version is unknown. string version = 2; // Additional attributes that describe the scope. [Optional]. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated KeyValue attributes = 3; - - // The number of attributes that were discarded. Attributes - // can be discarded because their keys are too long or because there are too many - // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 4; } - -// A reference to an Entity. -// Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs. -// -// Status: [Development] -message EntityRef { - // The Schema URL, if known. This is the identifier of the Schema that the entity data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // - // This schema_url applies to the data in this message and to the Resource attributes - // referenced by id_keys and description_keys. - // TODO: discuss if we are happy with this somewhat complicated definition of what - // the schema_url applies to. - // - // This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs. - string schema_url = 1; - - // Defines the type of the entity. MUST not change during the lifetime of the entity. - // For example: "service" or "host". This field is required and MUST not be empty - // for valid entities. - string type = 2; - - // Attribute Keys that identify the entity. - // MUST not change during the lifetime of the entity. The Id must contain at least one attribute. - // These keys MUST exist in the containing {message}.attributes. - repeated string id_keys = 3; - - // Descriptive (non-identifying) attribute keys of the entity. - // MAY change over the lifetime of the entity. MAY be empty. - // These attribute keys are not part of entity's identity. - // These keys MUST exist in the containing {message}.attributes. - repeated string description_keys = 4; -} \ No newline at end of file diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto index a6597d1e1148e..0b4b649729c9e 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto @@ -55,10 +55,6 @@ message ResourceLogs { // A list of ScopeLogs that originate from a resource. repeated ScopeLogs scope_logs = 2; - // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_logs" field which have their own schema_url field. string schema_url = 3; @@ -74,17 +70,13 @@ message ScopeLogs { // A list of log records. repeated LogRecord log_records = 2; - // The Schema URL, if known. This is the identifier of the Schema that the log data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "scope" field and all logs in the - // "log_records" field. + // This schema_url applies to all logs in the "logs" field. string schema_url = 3; } // Possible values for LogRecord.SeverityNumber. enum SeverityNumber { + // UNSPECIFIED is the default SeverityNumber, it MUST NOT be used. SEVERITY_NUMBER_UNSPECIFIED = 0; SEVERITY_NUMBER_TRACE = 1; SEVERITY_NUMBER_TRACE2 = 2; @@ -112,11 +104,9 @@ enum SeverityNumber { SEVERITY_NUMBER_FATAL4 = 24; } -// LogRecordFlags represents constants used to interpret the -// LogRecord.flags field, which is protobuf 'fixed32' type and is to -// be used as bit-fields. Each non-zero value defined in this enum is -// a bit-mask. To extract the bit-field, for example, use an -// expression like: +// LogRecordFlags is defined as a protobuf 'uint32' type and is to be used as +// bit-fields. Each non-zero value defined in this enum is a bit-mask. +// To extract the bit-field, for example, use an expression like: // // (logRecord.flags & LOG_RECORD_FLAGS_TRACE_FLAGS_MASK) // @@ -174,7 +164,6 @@ message LogRecord { // Additional attributes that describe the specific event occurrence. [Optional]. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 6; uint32 dropped_attributes_count = 7; @@ -211,16 +200,4 @@ message LogRecord { // - the field is not present, // - the field contains an invalid value. bytes span_id = 10; - - // A unique identifier of event category/type. - // All events with the same event_name are expected to conform to the same - // schema for both their attributes and their body. - // - // Recommended to be fully qualified and short (no longer than 256 characters). - // - // Presence of event_name on the log record identifies this record - // as an event. - // - // [Optional]. - string event_name = 12; } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto index a6fab4ee750f9..e85af8c3eae00 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/metrics/v1/metrics.proto @@ -29,24 +29,6 @@ option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1"; // storage, OR can be embedded by other protocols that transfer OTLP metrics // data but do not implement the OTLP protocol. // -// MetricsData -// └─── ResourceMetrics -// ├── Resource -// ├── SchemaURL -// └── ScopeMetrics -// ├── Scope -// ├── SchemaURL -// └── Metric -// ├── Name -// ├── Description -// ├── Unit -// └── data -// ├── Gauge -// ├── Sum -// ├── Histogram -// ├── ExponentialHistogram -// └── Summary -// // The main difference between this message and collector protocol is that // in this message there will not be any "control" or "metadata" specific to // OTLP protocol. @@ -73,10 +55,6 @@ message ResourceMetrics { // A list of metrics that originate from a resource. repeated ScopeMetrics scope_metrics = 2; - // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_metrics" field which have their own schema_url field. string schema_url = 3; @@ -92,12 +70,7 @@ message ScopeMetrics { // A list of metrics that originate from an instrumentation library. repeated Metric metrics = 2; - // The Schema URL, if known. This is the identifier of the Schema that the metric data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "scope" field and all metrics in the - // "metrics" field. + // This schema_url applies to all metrics in the "metrics" field. string schema_url = 3; } @@ -106,6 +79,7 @@ message ScopeMetrics { // // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md // +// // The data model and relation between entities is shown in the // diagram below. Here, "DataPoint" is the term used to refer to any // one of the specific data point value types, and "points" is the term used @@ -117,7 +91,7 @@ message ScopeMetrics { // - DataPoint contains timestamps, attributes, and one of the possible value type // fields. // -// Metric +// Metric // +------------+ // |name | // |description | @@ -188,18 +162,18 @@ message ScopeMetrics { message Metric { reserved 4, 6, 8; - // The name of the metric. + // name of the metric, including its DNS name prefix. It must be unique. string name = 1; - // A description of the metric, which can be used in documentation. + // description of the metric, which can be used in documentation. string description = 2; - // The unit in which the metric value is reported. Follows the format - // described by https://unitsofmeasure.org/ucum.html. + // unit in which the metric value is reported. Follows the format + // described by http://unitsofmeasure.org/ucum.html. string unit = 3; // Data determines the aggregation type (if any) of the metric, what is the - // reported value type for the data points, as well as the relatationship to + // reported value type for the data points, as well as the relationship to // the time interval over which they are reported. oneof data { Gauge gauge = 5; @@ -208,16 +182,6 @@ message Metric { ExponentialHistogram exponential_histogram = 10; Summary summary = 11; } - - // Additional metadata attributes that describe the metric. [Optional]. - // Attributes are non-identifying. - // Consumers SHOULD NOT need to be aware of these attributes. - // These attributes MAY be used to encode information allowing - // for lossless roundtrip translation to / from another data model. - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. - repeated opentelemetry.proto.common.v1.KeyValue metadata = 12; } // Gauge represents the type of a scalar metric that always exports the @@ -230,31 +194,25 @@ message Metric { // AggregationTemporality is not included. Consequently, this also means // "StartTimeUnixNano" is ignored for all data points. message Gauge { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). repeated NumberDataPoint data_points = 1; } // Sum represents the type of a scalar metric that is calculated as a sum of all // reported measurements over a time interval. message Sum { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). repeated NumberDataPoint data_points = 1; // aggregation_temporality describes if the aggregator reports delta changes // since last report time, or cumulative changes since a fixed start time. AggregationTemporality aggregation_temporality = 2; - // Represents whether the sum is monotonic. + // If "true" means that the sum is monotonic. bool is_monotonic = 3; } // Histogram represents the type of a metric that is calculated by aggregating // as a Histogram of all reported measurements over a time interval. message Histogram { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). repeated HistogramDataPoint data_points = 1; // aggregation_temporality describes if the aggregator reports delta changes @@ -265,8 +223,6 @@ message Histogram { // ExponentialHistogram represents the type of a metric that is calculated by aggregating // as a ExponentialHistogram of all reported double measurements over a time interval. message ExponentialHistogram { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). repeated ExponentialHistogramDataPoint data_points = 1; // aggregation_temporality describes if the aggregator reports delta changes @@ -276,16 +232,11 @@ message ExponentialHistogram { // Summary metric data are used to convey quantile summaries, // a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary) -// and OpenMetrics (see: https://github.com/prometheus/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) +// and OpenMetrics (see: https://github.com/OpenObservability/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) // data type. These data points cannot always be merged in a meaningful way. // While they can be useful in some applications, histogram data points are // recommended for new applications. -// Summary metrics do not have an aggregation temporality field. This is -// because the count and sum fields of a SummaryDataPoint are assumed to be -// cumulative values. message Summary { - // The time series data points. - // Note: Multiple time series may be included (same timestamp, different attributes). repeated SummaryDataPoint data_points = 1; } @@ -389,7 +340,6 @@ message NumberDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -438,7 +388,6 @@ message HistogramDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -466,7 +415,7 @@ message HistogramDataPoint { // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#histogram optional double sum = 5; // bucket_counts is an optional field contains the count values of histogram @@ -475,9 +424,7 @@ message HistogramDataPoint { // The sum of the bucket_counts must equal the value in the count field. // // The number of elements in bucket_counts array must be by one greater than - // the number of elements in explicit_bounds array. The exception to this rule - // is when the length of bucket_counts is 0, then the length of explicit_bounds - // must also be 0. + // the number of elements in explicit_bounds array. repeated fixed64 bucket_counts = 6; // explicit_bounds specifies buckets with explicitly defined bounds for values. @@ -493,9 +440,6 @@ message HistogramDataPoint { // Histogram buckets are inclusive of their upper boundary, except the last // bucket where the boundary is at infinity. This format is intentionally // compatible with the OpenMetrics histogram definition. - // - // If bucket_counts length is 0 then explicit_bounds length must also be 0, - // otherwise the data point is invalid. repeated double explicit_bounds = 7; // (Optional) List of exemplars collected from @@ -523,7 +467,6 @@ message ExponentialHistogramDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -539,19 +482,19 @@ message ExponentialHistogramDataPoint { // 1970. fixed64 time_unix_nano = 3; - // The number of values in the population. Must be + // count is the number of values in the population. Must be // non-negative. This value must be equal to the sum of the "bucket_counts" // values in the positive and negative Buckets plus the "zero_count" field. fixed64 count = 4; - // The sum of the values in the population. If count is zero then this field + // sum of the values in the population. If count is zero then this field // must be zero. // // Note: Sum should only be filled out when measuring non-negative discrete // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#histogram optional double sum = 5; // scale describes the resolution of the histogram. Boundaries are @@ -571,7 +514,7 @@ message ExponentialHistogramDataPoint { // values depend on the range of the data. sint32 scale = 6; - // The count of values that are either exactly zero or + // zero_count is the count of values that are either exactly zero or // within the region considered zero by the instrumentation at the // tolerated degree of precision. This bucket stores values that // cannot be expressed using the standard exponential formula as @@ -590,12 +533,12 @@ message ExponentialHistogramDataPoint { // Buckets are a set of bucket counts, encoded in a contiguous array // of counts. message Buckets { - // The bucket index of the first entry in the bucket_counts array. + // Offset is the bucket index of the first entry in the bucket_counts array. // // Note: This uses a varint encoding as a simple form of compression. sint32 offset = 1; - // An array of count values, where bucket_counts[i] carries + // bucket_counts is an array of count values, where bucket_counts[i] carries // the count of the bucket at index (offset+i). bucket_counts[i] is the count // of values greater than base^(offset+i) and less than or equal to // base^(offset+i+1). @@ -615,10 +558,10 @@ message ExponentialHistogramDataPoint { // measurements that were used to form the data point repeated Exemplar exemplars = 11; - // The minimum value over (start_time, end_time]. + // min is the minimum value over (start_time, end_time]. optional double min = 12; - // The maximum value over (start_time, end_time]. + // max is the maximum value over (start_time, end_time]. optional double max = 13; // ZeroThreshold may be optionally set to convey the width of the zero @@ -631,8 +574,7 @@ message ExponentialHistogramDataPoint { } // SummaryDataPoint is a single data point in a timeseries that describes the -// time-varying values of a Summary metric. The count and sum fields represent -// cumulative values. +// time-varying values of a Summary metric. message SummaryDataPoint { reserved 1; @@ -640,7 +582,6 @@ message SummaryDataPoint { // where this point belongs. The list may be empty (may contain 0 elements). // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; // StartTimeUnixNano is optional but strongly encouraged, see the @@ -666,7 +607,7 @@ message SummaryDataPoint { // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary + // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#summary double sum = 5; // Represents the value at a given quantile of a distribution. diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto index 42c5913cfae75..6637560bc3544 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto @@ -29,17 +29,9 @@ message Resource { // Set of attributes that describe the resource. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - // The number of dropped attributes. If the value is 0, then + // dropped_attributes_count is the number of dropped attributes. If the value is 0, then // no attributes were dropped. uint32 dropped_attributes_count = 2; - - // Set of entities that participate in this Resource. - // - // Note: keys in the references MUST exist in attributes of this message. - // - // Status: [Development] - repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3; } diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto index 8a992c148c642..864253ee41d87 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto @@ -55,10 +55,6 @@ message ResourceSpans { // A list of ScopeSpans that originate from a resource. repeated ScopeSpans scope_spans = 2; - // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_spans" field which have their own schema_url field. string schema_url = 3; @@ -74,12 +70,7 @@ message ScopeSpans { // A list of Spans that originate from an instrumentation scope. repeated Span spans = 2; - // The Schema URL, if known. This is the identifier of the Schema that the span data - // is recorded in. Notably, the last part of the URL path is the version number of the - // schema: http[s]://server[:port]/path/. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "scope" field and all spans and span - // events in the "spans" field. + // This schema_url applies to all spans and span events in the "spans" field. string schema_url = 3; } @@ -112,29 +103,6 @@ message Span { // field must be empty. The ID is an 8-byte array. bytes parent_span_id = 4; - // Flags, a bit field. - // - // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace - // Context specification. To read the 8-bit W3C trace flag, use - // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. - // - // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. - // - // Bits 8 and 9 represent the 3 states of whether a span's parent - // is remote. The states are (unknown, is not remote, is remote). - // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. - // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. - // - // When creating span messages, if the message is logically forwarded from another source - // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD - // be copied as-is. If creating from a source that does not have an equivalent flags field - // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST - // be set to zero. - // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. - // - // [Optional]. - fixed32 flags = 16; - // A description of the span's operation. // // For example, the name can be a qualified method name or a file name @@ -183,7 +151,7 @@ message Span { // and `SERVER` (callee) to identify queueing latency associated with the span. SpanKind kind = 6; - // The start time of the span. On the client side, this is the time + // start_time_unix_nano is the start time of the span. On the client side, this is the time // kept by the local machine where the span execution starts. On the server side, this // is the time when the server's application handler starts running. // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. @@ -191,7 +159,7 @@ message Span { // This field is semantically required and it is expected that end_time >= start_time. fixed64 start_time_unix_nano = 7; - // The end time of the span. On the client side, this is the time + // end_time_unix_nano is the end time of the span. On the client side, this is the time // kept by the local machine where the span execution ends. On the server side, this // is the time when the server application handler stops running. // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. @@ -199,20 +167,21 @@ message Span { // This field is semantically required and it is expected that end_time >= start_time. fixed64 end_time_unix_nano = 8; - // A collection of key/value pairs. Note, global attributes + // attributes is a collection of key/value pairs. Note, global attributes // like server name can be set using the resource API. Examples of attributes: // - // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" - // "/http/server_latency": 300 - // "example.com/myattribute": true - // "example.com/score": 10.239 + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "example.com/my_attribute": true + // "example.com/score": 10.239 // + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; - // The number of attributes that were discarded. Attributes + // dropped_attributes_count is the number of attributes that were discarded. Attributes // can be discarded because their keys are too long or because there are too many // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 10; @@ -220,28 +189,27 @@ message Span { // Event is a time-stamped annotation of the span, consisting of user-supplied // text description and key-value pairs. message Event { - // The time the event occurred. + // time_unix_nano is the time the event occurred. fixed64 time_unix_nano = 1; - // The name of the event. + // name of the event. // This field is semantically required to be set to non-empty string. string name = 2; - // A collection of attribute key/value pairs on the event. + // attributes is a collection of attribute key/value pairs on the event. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; - // The number of dropped attributes. If the value is 0, + // dropped_attributes_count is the number of dropped attributes. If the value is 0, // then no attributes were dropped. uint32 dropped_attributes_count = 4; } - // A collection of Event items. + // events is a collection of Event items. repeated Event events = 11; - // The number of dropped events. If the value is 0, then no + // dropped_events_count is the number of dropped events. If the value is 0, then no // events were dropped. uint32 dropped_events_count = 12; @@ -260,41 +228,21 @@ message Span { // The trace_state associated with the link. string trace_state = 3; - // A collection of attribute key/value pairs on the link. + // attributes is a collection of attribute key/value pairs on the link. // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). - // The behavior of software that receives duplicated keys can be unpredictable. repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; - // The number of dropped attributes. If the value is 0, + // dropped_attributes_count is the number of dropped attributes. If the value is 0, // then no attributes were dropped. uint32 dropped_attributes_count = 5; - - // Flags, a bit field. - // - // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace - // Context specification. To read the 8-bit W3C trace flag, use - // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. - // - // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. - // - // Bits 8 and 9 represent the 3 states of whether the link is remote. - // The states are (unknown, is not remote, is remote). - // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. - // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. - // - // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. - // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. - // - // [Optional]. - fixed32 flags = 6; } - // A collection of Links, which are references from this span to a span + // links is a collection of Links, which are references from this span to a span // in the same or different trace. repeated Link links = 13; - // The number of dropped links after the maximum size was + // dropped_links_count is the number of dropped links after the maximum size was // enforced. If this value is 0, then no links were dropped. uint32 dropped_links_count = 14; @@ -316,7 +264,7 @@ message Status { enum StatusCode { // The default status. STATUS_CODE_UNSET = 0; - // The Span has been validated by an Application developer or Operator to + // The Span has been validated by an Application developer or Operator to // have completed successfully. STATUS_CODE_OK = 1; // The Span contains an error. @@ -326,34 +274,3 @@ message Status { // The status code. StatusCode code = 3; } - -// SpanFlags represents constants used to interpret the -// Span.flags field, which is protobuf 'fixed32' type and is to -// be used as bit-fields. Each non-zero value defined in this enum is -// a bit-mask. To extract the bit-field, for example, use an -// expression like: -// -// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) -// -// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. -// -// Note that Span flags were introduced in version 1.1 of the -// OpenTelemetry protocol. Older Span producers do not set this -// field, consequently consumers should not rely on the absence of a -// particular flag bit to indicate the presence of a particular feature. -enum SpanFlags { - // The zero value for the enum. Should not be used for comparisons. - // Instead use bitwise "and" with the appropriate mask as shown above. - SPAN_FLAGS_DO_NOT_USE = 0; - - // Bits 0-7 are used for trace flags. - SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; - - // Bits 8 and 9 are used to indicate that the parent span or link span is remote. - // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. - // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. - SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; - SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; - - // Bits 10-31 are reserved for future use. -} diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index 10c5bfde42745..9fb0bac71a1cd 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -56,10 +56,8 @@ fn create_test_logs_request() -> Request { Request::new(ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { resource: Some(OtelResource { - entity_refs: vec![], attributes: vec![KeyValue { key: "res_key".into(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("res_val".into())), }), @@ -72,7 +70,6 @@ fn create_test_logs_request() -> Request { version: "1.2.3".into(), attributes: vec![KeyValue { key: "scope_attr".into(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("scope_val".into())), }), @@ -82,7 +79,6 @@ fn create_test_logs_request() -> Request { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, - event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { @@ -90,7 +86,6 @@ fn create_test_logs_request() -> Request { }), attributes: vec![KeyValue { key: "attr_key".into(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("attr_val".into())), }), @@ -112,10 +107,8 @@ fn create_test_metrics_request() -> ExportMetricsServiceRequest { ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -135,20 +128,17 @@ fn create_test_metrics_request() -> ExportMetricsServiceRequest { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::Summary(Summary { data_points: vec![SummaryDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -201,7 +191,6 @@ fn create_test_traces_request() -> ExportTraceServiceRequest { dropped_events_count: 0, links: vec![], dropped_links_count: 0, - flags: 0, status: None, trace_state: "".to_string(), }], @@ -393,10 +382,8 @@ async fn receive_sum_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -416,19 +403,16 @@ async fn receive_sum_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::Sum(Sum { data_points: vec![NumberDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -490,10 +474,8 @@ async fn receive_sum_non_monotonic_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -513,19 +495,16 @@ async fn receive_sum_non_monotonic_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::Sum(Sum { data_points: vec![NumberDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -587,10 +566,8 @@ async fn receive_gauge_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -610,19 +587,16 @@ async fn receive_gauge_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::Gauge(Gauge { data_points: vec![NumberDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -681,10 +655,8 @@ async fn receive_histogram_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -704,21 +676,18 @@ async fn receive_histogram_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::Histogram(Histogram { aggregation_temporality: AggregationTemporality::Cumulative as i32, data_points: vec![HistogramDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -813,10 +782,8 @@ async fn receive_histogram_delta_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -836,21 +803,18 @@ async fn receive_histogram_delta_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::Histogram(Histogram { aggregation_temporality: AggregationTemporality::Delta as i32, data_points: vec![HistogramDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -945,10 +909,8 @@ async fn receive_expontential_histogram_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -968,21 +930,18 @@ async fn receive_expontential_histogram_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::ExponentialHistogram(ExponentialHistogram { aggregation_temporality: AggregationTemporality::Cumulative as i32, data_points: vec![ExponentialHistogramDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -1090,10 +1049,8 @@ async fn receive_summary_metric() { let req = Request::new(ExportMetricsServiceRequest { resource_metrics: vec![ResourceMetrics { resource: Some(Resource { - entity_refs: vec![], attributes: vec![KeyValue { key: "service.name".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("vector-collector".to_string())), }), @@ -1113,20 +1070,17 @@ async fn receive_summary_metric() { name: "some.random.metric".to_string(), description: "Some random metric we use for test".to_string(), unit: "1".to_string(), - metadata: vec![], data: Some(Data::Summary(Summary { data_points: vec![SummaryDataPoint { attributes: vec![ KeyValue { key: "host".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue("localhost".to_string())), }), }, KeyValue { key: "service".to_string(), - key_strindex: 0, value: Some(AnyValue { value: Some(StringValue( "vector-collector".to_string(), @@ -1300,7 +1254,6 @@ async fn http_headers_logs_use_otlp_decoding_false() { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, - event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { @@ -1382,7 +1335,6 @@ async fn http_headers_logs_use_otlp_decoding_true() { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, - event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { @@ -1666,7 +1618,6 @@ async fn http_logs_use_otlp_decoding_emits_metric() { log_records: vec![LogRecord { time_unix_nano: 1, observed_time_unix_nano: 2, - event_name: String::new(), severity_number: 9, severity_text: "info".into(), body: Some(AnyValue { From aad30ebe0739e10f7af400940e044f477d7dc6c1 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Thu, 28 May 2026 17:56:01 -0400 Subject: [PATCH 6/9] improved coverage and efficiency --- src/transforms/policy/adapter.rs | 113 +++++---- src/transforms/policy/internal_events.rs | 38 ++- src/transforms/policy/otlp_adapter.rs | 43 ++-- src/transforms/policy/otlp_metric_adapter.rs | 253 ++++++++++++++++--- src/transforms/policy/otlp_trace_adapter.rs | 252 +++++++++++++++++- src/transforms/policy/tests.rs | 228 ++++++++++++++++- src/transforms/policy/transform.rs | 6 +- 7 files changed, 805 insertions(+), 128 deletions(-) diff --git a/src/transforms/policy/adapter.rs b/src/transforms/policy/adapter.rs index 94940217099de..15f85bb01e5e0 100644 --- a/src/transforms/policy/adapter.rs +++ b/src/transforms/policy/adapter.rs @@ -16,7 +16,7 @@ use policy_rs::proto::tero::policy::v1::LogField; use policy_rs::{LogFieldSelector, Matchable, Transformable, engine::LogSignal}; use vector_lib::{ event::{LogEvent, Value}, - lookup::{OwnedTargetPath, OwnedValuePath, lookup_v2::ConfigValuePath}, + lookup::{OwnedValuePath, PathPrefix}, }; use super::field_mapping::FieldMapping; @@ -32,93 +32,97 @@ impl<'a> VectorLogAdapter<'a> { pub const fn new(log: &'a mut LogEvent, mapping: &'a FieldMapping) -> Self { Self { log, mapping } } +} - /// Resolve a `LogFieldSelector` to the corresponding `LogEvent` path. - /// - /// Returns `None` for selectors that the current mapping does not - /// represent — for example, the protobuf-default `LogField::Unspecified` - /// variant or fields that haven't been wired up yet. The engine treats - /// `None` as "field not present", which is the correct fail-soft - /// behaviour for matching. - fn path_for(&self, selector: &LogFieldSelector) -> Option { - match selector { - LogFieldSelector::Simple(field) => self.simple_path(*field), - LogFieldSelector::LogAttribute(path) => Some(FieldMapping::append_segments( - &self.mapping.log_attributes, - path, - )), - LogFieldSelector::ResourceAttribute(path) => Some(FieldMapping::append_segments( - &self.mapping.resource_attributes, - path, - )), - LogFieldSelector::ScopeAttribute(path) => Some(FieldMapping::append_segments( - &self.mapping.scope_attributes, - path, - )), - } +/// Resolve a `LogFieldSelector` to the corresponding `LogEvent` value path. +/// +/// Simple fields borrow the mapping's stored path (`Cow::Borrowed`, no +/// allocation); only attribute selectors — which append dynamic segments — +/// build a fresh path (`Cow::Owned`). +/// +/// Returns `None` for selectors the mapping doesn't represent (e.g. the +/// protobuf-default `LogField::Unspecified`); the engine treats `None` as +/// "field not present", the correct fail-soft behaviour for matching. +/// +/// Takes `&FieldMapping` rather than `&self` so callers can hold the resulting +/// borrow of the mapping while mutating `self.log` (disjoint fields). +fn value_path<'m>( + mapping: &'m FieldMapping, + selector: &LogFieldSelector, +) -> Option> { + match selector { + LogFieldSelector::Simple(field) => simple_value_path(mapping, *field).map(Cow::Borrowed), + LogFieldSelector::LogAttribute(path) => Some(Cow::Owned(FieldMapping::append_segments( + &mapping.log_attributes, + path, + ))), + LogFieldSelector::ResourceAttribute(path) => Some(Cow::Owned( + FieldMapping::append_segments(&mapping.resource_attributes, path), + )), + LogFieldSelector::ScopeAttribute(path) => Some(Cow::Owned(FieldMapping::append_segments( + &mapping.scope_attributes, + path, + ))), } +} - fn simple_path(&self, field: LogField) -> Option { - let mapped: &ConfigValuePath = match field { - LogField::Body => &self.mapping.body, - LogField::SeverityText => &self.mapping.severity_text, - LogField::TraceId => &self.mapping.trace_id, - LogField::SpanId => &self.mapping.span_id, - LogField::EventName => &self.mapping.event_name, - LogField::ResourceSchemaUrl => &self.mapping.resource_schema_url, - LogField::ScopeSchemaUrl => &self.mapping.scope_schema_url, - LogField::Unspecified => return None, - }; - Some(mapped.0.clone()) - } +fn simple_value_path(mapping: &FieldMapping, field: LogField) -> Option<&OwnedValuePath> { + Some(match field { + LogField::Body => &mapping.body.0, + LogField::SeverityText => &mapping.severity_text.0, + LogField::TraceId => &mapping.trace_id.0, + LogField::SpanId => &mapping.span_id.0, + LogField::EventName => &mapping.event_name.0, + LogField::ResourceSchemaUrl => &mapping.resource_schema_url.0, + LogField::ScopeSchemaUrl => &mapping.scope_schema_url.0, + LogField::Unspecified => return None, + }) } impl Matchable for VectorLogAdapter<'_> { type Signal = LogSignal; fn get_field(&self, field: &LogFieldSelector) -> Option> { - let path = self.path_for(field)?; - let target = OwnedTargetPath::event(path); - let value = self.log.get(&target)?; + let path = value_path(self.mapping, field)?; + let value = self.log.get((PathPrefix::Event, path.as_ref()))?; value_to_match_string(value) } fn field_exists(&self, field: &LogFieldSelector) -> bool { - let Some(path) = self.path_for(field) else { + let Some(path) = value_path(self.mapping, field) else { return false; }; - let target = OwnedTargetPath::event(path); - self.log.get(&target).is_some() + self.log.get((PathPrefix::Event, path.as_ref())).is_some() } } impl Transformable for VectorLogAdapter<'_> { fn set_field(&mut self, field: &LogFieldSelector, value: &str) { - if let Some(path) = self.path_for(field) { - let target = OwnedTargetPath::event(path); - self.log.insert(&target, value.to_string()); + if let Some(path) = value_path(self.mapping, field) { + self.log + .insert((PathPrefix::Event, path.as_ref()), value.to_string()); } } fn delete_field(&mut self, field: &LogFieldSelector) -> bool { - let Some(path) = self.path_for(field) else { + let Some(path) = value_path(self.mapping, field) else { return false; }; - let target = OwnedTargetPath::event(path); - self.log.remove(&target).is_some() + self.log.remove((PathPrefix::Event, path.as_ref())).is_some() } fn move_field(&mut self, from: &LogFieldSelector, to: &LogFieldSelector) { // The engine guarantees `from` exists and `to` does not, so we can // remove from the source and unconditionally insert at the target // without losing data on a name collision. - let (Some(from_path), Some(to_path)) = (self.path_for(from), self.path_for(to)) else { + let (Some(from_path), Some(to_path)) = + (value_path(self.mapping, from), value_path(self.mapping, to)) + else { return; }; - let from_target = OwnedTargetPath::event(from_path); - let to_target = OwnedTargetPath::event(to_path); - if let Some(value) = self.log.remove(&from_target) { - self.log.insert(&to_target, value); + if let Some(value) = self.log.remove((PathPrefix::Event, from_path.as_ref())) { + self.log + .insert((PathPrefix::Event, to_path.as_ref()), value); } } } @@ -150,6 +154,7 @@ mod tests { use chrono::Utc; use policy_rs::proto::tero::policy::v1::LogField; use vector_lib::event::LogEvent; + use vector_lib::lookup::OwnedTargetPath; fn make_log() -> LogEvent { let mut log = LogEvent::default(); diff --git a/src/transforms/policy/internal_events.rs b/src/transforms/policy/internal_events.rs index 3fda3416bc5ac..c460e8a01d4bf 100644 --- a/src/transforms/policy/internal_events.rs +++ b/src/transforms/policy/internal_events.rs @@ -28,9 +28,43 @@ impl DropReason { } } -pub(crate) fn emit_dropped(reason: DropReason) { +/// Per-envelope accumulator for dropped-record counts. +/// +/// OTLP envelopes can drop many records in a single event; emitting one +/// `ComponentEventsDropped` per record would hammer the metrics path. Instead +/// callers `record` each drop and `emit` once per envelope, collapsing N +/// `emit!` calls into at most one per reason. +#[derive(Default)] +pub(crate) struct DropCounts { + policy_drop: u64, + sample_rejected: u64, + rate_limited: u64, +} + +impl DropCounts { + pub(crate) fn record(&mut self, reason: DropReason) { + match reason { + DropReason::PolicyDrop => self.policy_drop += 1, + DropReason::SampleRejected => self.sample_rejected += 1, + DropReason::RateLimited => self.rate_limited += 1, + } + } + + pub(crate) fn emit(&self) { + emit_dropped(DropReason::PolicyDrop, self.policy_drop); + emit_dropped(DropReason::SampleRejected, self.sample_rejected); + emit_dropped(DropReason::RateLimited, self.rate_limited); + } +} + +/// Emit a single aggregated `ComponentEventsDropped` for `count` records. A +/// zero count emits nothing. +pub(crate) fn emit_dropped(reason: DropReason, count: u64) { + if count == 0 { + return; + } emit!(ComponentEventsDropped:: { - count: 1, + count: count as usize, reason: reason.as_str(), }); } diff --git a/src/transforms/policy/otlp_adapter.rs b/src/transforms/policy/otlp_adapter.rs index bdfd4ff064d14..ab6c35e56cf2a 100644 --- a/src/transforms/policy/otlp_adapter.rs +++ b/src/transforms/policy/otlp_adapter.rs @@ -38,7 +38,7 @@ use policy_rs::{ }; use vector_lib::event::{LogEvent, ObjectMap, Value}; -use super::internal_events::{DropReason, emit_dropped}; +use super::internal_events::{DropCounts, DropReason}; /// Iterate every record inside an OTLP envelope event, applying policies /// per-record. Returns `Some(log)` to forward (with mutated and possibly @@ -59,6 +59,7 @@ pub(super) async fn evaluate_envelope( return Some(log); }; + let mut drops = DropCounts::default(); let mut i = 0; while i < resource_logs.len() { // Lift `resource` out of the entry so the adapter can hold a mutable @@ -112,15 +113,15 @@ pub(super) async fn evaluate_envelope( } Ok(EvaluateResult::Drop { .. }) => { records.remove(k); - emit_dropped(DropReason::PolicyDrop); + drops.record(DropReason::PolicyDrop); } Ok(EvaluateResult::Sample { keep: false, .. }) => { records.remove(k); - emit_dropped(DropReason::SampleRejected); + drops.record(DropReason::SampleRejected); } Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { records.remove(k); - emit_dropped(DropReason::RateLimited); + drops.record(DropReason::RateLimited); } Err(error) => { error!( @@ -164,6 +165,8 @@ pub(super) async fn evaluate_envelope( } } + drops.emit(); + if resource_logs.is_empty() { None } else { @@ -470,7 +473,7 @@ pub(super) fn find_attribute_path<'a>( fn find_in_kvlist<'a>(attrs: &'a [Value], path: &[String]) -> Option> { let first = path.first()?; for kv in attrs { - if attribute_key(kv) != Some(first.as_str()) { + if !attribute_key_eq(kv, first) { continue; } let value = kv.as_object().and_then(|o| o.get("value")); @@ -494,7 +497,7 @@ fn exists_in_kvlist(attrs: &[Value], path: &[String]) -> bool { return false; }; for kv in attrs { - if attribute_key(kv) != Some(first.as_str()) { + if !attribute_key_eq(kv, first) { continue; } let value = kv.as_object().and_then(|o| o.get("value")); @@ -518,13 +521,14 @@ fn nested_values(value: Option<&Value>) -> Option<&[Value]> { .and_then(Value::as_array) } -fn attribute_key(item: &Value) -> Option<&str> { - item.as_object() - .and_then(|o| o.get("key")) - .and_then(|v| match v { - Value::Bytes(b) => std::str::from_utf8(b).ok(), - _ => None, - }) +/// Whether an attribute entry's `key` equals `key`, comparing raw bytes so we +/// skip UTF-8 validation on every entry of a linear scan (the key is an OTLP +/// `string`, but for an equality test the byte representation is sufficient). +fn attribute_key_eq(item: &Value, key: &str) -> bool { + matches!( + item.as_object().and_then(|o| o.get("key")), + Some(Value::Bytes(b)) if b.as_ref() == key.as_bytes() + ) } // ============================================================================= @@ -536,10 +540,7 @@ fn set_string_attr(attrs: &mut Vec, path: &[String], value: &str) { let Some(key) = path.first() else { return; }; - if let Some(entry) = attrs - .iter_mut() - .find(|kv| attribute_key(kv) == Some(key.as_str())) - { + if let Some(entry) = attrs.iter_mut().find(|kv| attribute_key_eq(kv, key)) { if let Some(obj) = entry.as_object_mut() { obj.insert("value".into(), make_string_any_value(value)); } @@ -554,22 +555,20 @@ fn remove_attr(attrs: &mut Vec, path: &[String]) -> bool { return false; }; let before = attrs.len(); - attrs.retain(|kv| attribute_key(kv) != Some(key.as_str())); + attrs.retain(|kv| !attribute_key_eq(kv, key)); attrs.len() < before } /// Remove and return the first-segment attribute entry, preserving its value. fn remove_attr_kv(attrs: &mut Vec, path: &[String]) -> Option { let key = path.first()?; - let idx = attrs - .iter() - .position(|kv| attribute_key(kv) == Some(key.as_str()))?; + let idx = attrs.iter().position(|kv| attribute_key_eq(kv, key))?; Some(attrs.remove(idx)) } /// Insert an entry under `key`, removing any existing entry with that key first. fn upsert_entry(attrs: &mut Vec, key: &str, entry: Value) { - attrs.retain(|kv| attribute_key(kv) != Some(key)); + attrs.retain(|kv| !attribute_key_eq(kv, key)); attrs.push(entry); } diff --git a/src/transforms/policy/otlp_metric_adapter.rs b/src/transforms/policy/otlp_metric_adapter.rs index d1a464d6f84b4..04b4229e2313b 100644 --- a/src/transforms/policy/otlp_metric_adapter.rs +++ b/src/transforms/policy/otlp_metric_adapter.rs @@ -46,67 +46,85 @@ pub(super) async fn evaluate_metrics_envelope( return Some(log); }; + let mut dropped = 0u64; let mut i = 0; while i < resource_metrics.len() { - // Metrics are read-only, so immutable snapshots of resource/scope are - // enough (no remove/re-insert dance needed). - let resource = resource_metrics[i].get("resource").cloned(); - let resource_schema_url = resource_metrics[i].get("schemaUrl").cloned(); - - let mut prune_rm = false; + // Phase 1 — score every metric. Metric evaluation is filter-only (no + // transform), so resource/scope/metric are all read-only here: we can + // borrow them immutably and avoid both the deep clone and the + // remove/re-insert lift the log/trace paths need. `keep[s][m]` is + // whether scope `s`'s metric `m` survives. + let keep: Vec> = { + let rm = &resource_metrics[i]; + let resource = rm.get("resource"); + let resource_schema_url = rm.get("schemaUrl"); + let mut per_scope = Vec::new(); + if let Some(scope_metrics) = rm.get("scopeMetrics").and_then(Value::as_array) { + for sm in scope_metrics { + let scope = sm.get("scope"); + let scope_schema_url = sm.get("schemaUrl"); + let mut per_metric = Vec::new(); + if let Some(metrics) = sm.get("metrics").and_then(Value::as_array) { + for metric in metrics { + let adapter = MetricAdapter { + metric, + resource, + scope, + resource_schema_url, + scope_schema_url, + }; + let drop = matches!( + engine.evaluate(snapshot, &adapter).await, + Ok(EvaluateResult::Drop { .. }) + ); + if drop { + dropped += 1; + } + per_metric.push(!drop); + } + } + per_scope.push(per_metric); + } + } + per_scope + }; + // Phase 2 — apply the decisions, then prune emptied scopes / resource. if let Some(scope_metrics) = resource_metrics[i] .get_mut("scopeMetrics") .and_then(Value::as_array_mut) { + // `scope_idx` indexes `keep` by original scope position; `j` tracks + // the live position as emptied scopes are removed. + let mut scope_idx = 0; let mut j = 0; while j < scope_metrics.len() { - let scope = scope_metrics[j].get("scope").cloned(); - let scope_schema_url = scope_metrics[j].get("schemaUrl").cloned(); - - let mut prune_sm = false; - + let scope_keep = &keep[scope_idx]; + scope_idx += 1; if let Some(metrics) = scope_metrics[j] .get_mut("metrics") .and_then(Value::as_array_mut) { - // Evaluate first (immutable borrows), then retain by index. - let mut keep = Vec::with_capacity(metrics.len()); - for metric in metrics.iter() { - let adapter = MetricAdapter { - metric, - resource: resource.as_ref(), - scope: scope.as_ref(), - resource_schema_url: resource_schema_url.as_ref(), - scope_schema_url: scope_schema_url.as_ref(), - }; - let drop = matches!( - engine.evaluate(snapshot, &adapter).await, - Ok(EvaluateResult::Drop { .. }) - ); - if drop { - emit_dropped(DropReason::PolicyDrop); - } - keep.push(!drop); - } - let mut idx = 0; + let mut m = 0; metrics.retain(|_| { - let k = keep[idx]; - idx += 1; + let k = scope_keep[m]; + m += 1; k }); - prune_sm = metrics.is_empty(); - } - - if prune_sm { - scope_metrics.remove(j); - } else { - j += 1; + if metrics.is_empty() { + scope_metrics.remove(j); + continue; + } } + j += 1; } - prune_rm = scope_metrics.is_empty(); } + let prune_rm = resource_metrics[i] + .get("scopeMetrics") + .and_then(Value::as_array) + .map(<[Value]>::is_empty) + .unwrap_or(true); if prune_rm { resource_metrics.remove(i); } else { @@ -114,6 +132,8 @@ pub(super) async fn evaluate_metrics_envelope( } } + emit_dropped(DropReason::PolicyDrop, dropped); + if resource_metrics.is_empty() { None } else { @@ -215,3 +235,154 @@ impl Matchable for MetricAdapter<'_> { } } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn val(value: serde_json::Value) -> Value { + Value::from(value) + } + + fn adapter<'a>( + metric: &'a Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + ) -> MetricAdapter<'a> { + MetricAdapter { + metric, + resource, + scope, + resource_schema_url: None, + scope_schema_url: None, + } + } + + fn get(metric: &Value, sel: MetricFieldSelector) -> Option { + adapter(metric, None, None) + .get_field(&sel) + .map(Cow::into_owned) + } + + #[test] + fn metric_type_derived_from_data_variant() { + let cases = [ + (json!({"name": "m", "gauge": {"dataPoints": []}}), "METRIC_TYPE_GAUGE"), + (json!({"name": "m", "sum": {"dataPoints": []}}), "METRIC_TYPE_SUM"), + ( + json!({"name": "m", "histogram": {"dataPoints": []}}), + "METRIC_TYPE_HISTOGRAM", + ), + ( + json!({"name": "m", "exponentialHistogram": {"dataPoints": []}}), + "METRIC_TYPE_EXPONENTIAL_HISTOGRAM", + ), + ( + json!({"name": "m", "summary": {"dataPoints": []}}), + "METRIC_TYPE_SUMMARY", + ), + ]; + for (metric, expected) in cases { + let metric = val(metric); + assert_eq!( + get(&metric, MetricFieldSelector::Type), + Some(expected.to_string()), + ); + } + } + + #[test] + fn temporality_read_from_data_variant() { + let delta = val(json!({ + "name": "m", + "sum": {"dataPoints": [], "aggregationTemporality": "AGGREGATION_TEMPORALITY_DELTA"} + })); + assert_eq!( + get(&delta, MetricFieldSelector::Temporality), + Some("AGGREGATION_TEMPORALITY_DELTA".to_string()), + ); + + // Gauge has no temporality. + let gauge = val(json!({"name": "m", "gauge": {"dataPoints": []}})); + assert_eq!(get(&gauge, MetricFieldSelector::Temporality), None); + } + + #[test] + fn descriptor_fields() { + let m = val(json!({ + "name": "http.requests", "description": "count", "unit": "1", + "gauge": {"dataPoints": []} + })); + assert_eq!( + get(&m, MetricFieldSelector::Simple(MetricField::Name)), + Some("http.requests".to_string()), + ); + assert_eq!( + get(&m, MetricFieldSelector::Simple(MetricField::Description)), + Some("count".to_string()), + ); + assert_eq!( + get(&m, MetricFieldSelector::Simple(MetricField::Unit)), + Some("1".to_string()), + ); + } + + #[test] + fn datapoint_attribute_uses_first_datapoint() { + let m = val(json!({"name": "m", "sum": {"dataPoints": [ + {"attributes": [{"key": "http.method", "value": {"stringValue": "GET"}}]}, + {"attributes": [{"key": "http.method", "value": {"stringValue": "POST"}}]} + ]}})); + assert_eq!( + get( + &m, + MetricFieldSelector::DatapointAttribute(vec!["http.method".to_string()]) + ), + Some("GET".to_string()), + ); + } + + #[test] + fn non_string_datapoint_attr_exists_but_is_unmatchable() { + let m = val(json!({"name": "m", "sum": {"dataPoints": [ + {"attributes": [{"key": "count", "value": {"intValue": "42"}}]} + ]}})); + let sel = MetricFieldSelector::DatapointAttribute(vec!["count".to_string()]); + // Not coercible to a string for matching... + assert_eq!(get(&m, sel.clone()), None); + // ...but `exists` still fires. + assert!(adapter(&m, None, None).field_exists(&sel)); + } + + #[test] + fn resource_and_scope_fields() { + let m = val(json!({"name": "m", "gauge": {"dataPoints": []}})); + let resource = + val(json!({"attributes": [{"key": "service.name", "value": {"stringValue": "api"}}]})); + let scope = val(json!({ + "name": "lib", "version": "1.2", + "attributes": [{"key": "k", "value": {"stringValue": "x"}}] + })); + let adapter = adapter(&m, Some(&resource), Some(&scope)); + let g = |sel| adapter.get_field(&sel).map(Cow::into_owned); + assert_eq!( + g(MetricFieldSelector::ResourceAttribute(vec![ + "service.name".to_string() + ])), + Some("api".to_string()), + ); + assert_eq!( + g(MetricFieldSelector::ScopeAttribute(vec!["k".to_string()])), + Some("x".to_string()), + ); + assert_eq!( + g(MetricFieldSelector::Simple(MetricField::ScopeName)), + Some("lib".to_string()), + ); + assert_eq!( + g(MetricFieldSelector::Simple(MetricField::ScopeVersion)), + Some("1.2".to_string()), + ); + } +} diff --git a/src/transforms/policy/otlp_trace_adapter.rs b/src/transforms/policy/otlp_trace_adapter.rs index ae1e295f94af8..72dccff46634a 100644 --- a/src/transforms/policy/otlp_trace_adapter.rs +++ b/src/transforms/policy/otlp_trace_adapter.rs @@ -20,7 +20,7 @@ use policy_rs::{ }; use vector_lib::event::{TraceEvent, Value}; -use super::internal_events::{DropReason, emit_dropped}; +use super::internal_events::{DropCounts, DropReason}; use super::otlp_adapter::{attribute_exists_path, find_attribute_path, non_empty}; /// Iterate every span in an OTLP traces envelope, sampling/dropping in place. @@ -40,9 +40,14 @@ pub(super) async fn evaluate_traces_envelope( return true; }; + let mut drops = DropCounts::default(); let mut i = 0; while i < resource_spans.len() { - let resource = resource_spans[i].get("resource").cloned(); + // Lift resource/scope out (a move, not a clone) so the adapter can read + // them while we mutate the sibling `spans` array; re-attached after. + let resource = resource_spans[i] + .as_object_mut() + .and_then(|o| o.remove("resource")); let resource_schema_url = resource_spans[i].get("schemaUrl").cloned(); let mut prune_rs = false; @@ -53,7 +58,9 @@ pub(super) async fn evaluate_traces_envelope( { let mut j = 0; while j < scope_spans.len() { - let scope = scope_spans[j].get("scope").cloned(); + let scope = scope_spans[j] + .as_object_mut() + .and_then(|o| o.remove("scope")); let scope_schema_url = scope_spans[j].get("schemaUrl").cloned(); let mut prune_ss = false; @@ -77,11 +84,11 @@ pub(super) async fn evaluate_traces_envelope( let keep = match result { Ok(EvaluateResult::Drop { .. }) => { - emit_dropped(DropReason::PolicyDrop); + drops.record(DropReason::PolicyDrop); false } Ok(EvaluateResult::Sample { keep: false, .. }) => { - emit_dropped(DropReason::SampleRejected); + drops.record(DropReason::SampleRejected); false } Ok(_) => true, @@ -103,6 +110,13 @@ pub(super) async fn evaluate_traces_envelope( prune_ss = spans.is_empty(); } + // Re-attach the (read-only) scope. + if let Some(scope) = scope + && let Some(obj) = scope_spans[j].as_object_mut() + { + obj.insert("scope".into(), scope); + } + if prune_ss { scope_spans.remove(j); } else { @@ -112,6 +126,13 @@ pub(super) async fn evaluate_traces_envelope( prune_rs = scope_spans.is_empty(); } + // Re-attach the (read-only) resource. + if let Some(resource) = resource + && let Some(obj) = resource_spans[i].as_object_mut() + { + obj.insert("resource".into(), resource); + } + if prune_rs { resource_spans.remove(i); } else { @@ -119,6 +140,7 @@ pub(super) async fn evaluate_traces_envelope( } } + drops.emit(); !resource_spans.is_empty() } @@ -305,3 +327,223 @@ fn merge_ot_tracestate(tracestate: &str, sub_kv: &str) -> String { } result } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn val(value: serde_json::Value) -> Value { + Value::from(value) + } + + /// Build a read-only adapter over an owned span value. + fn get(span: serde_json::Value, sel: TraceFieldSelector) -> Option { + let mut span = val(span); + let adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.get_field(&sel).map(Cow::into_owned) + } + + fn exists(span: serde_json::Value, sel: &TraceFieldSelector) -> bool { + let mut span = val(span); + let adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.field_exists(sel) + } + + #[test] + fn simple_span_fields() { + let span = json!({ + "name": "GET /x", "traceId": "abc", "spanId": "def", + "parentSpanId": "p", "traceState": "ot=th:8", "kind": "SPAN_KIND_SERVER" + }); + assert_eq!( + get(span.clone(), TraceFieldSelector::Simple(TraceField::Name)), + Some("GET /x".to_string()), + ); + assert_eq!( + get(span.clone(), TraceFieldSelector::Simple(TraceField::TraceId)), + Some("abc".to_string()), + ); + assert_eq!( + get(span.clone(), TraceFieldSelector::Simple(TraceField::SpanId)), + Some("def".to_string()), + ); + assert_eq!( + get( + span.clone(), + TraceFieldSelector::Simple(TraceField::ParentSpanId) + ), + Some("p".to_string()), + ); + assert_eq!( + get( + span.clone(), + TraceFieldSelector::Simple(TraceField::TraceState) + ), + Some("ot=th:8".to_string()), + ); + assert_eq!( + get(span, TraceFieldSelector::SpanKind), + Some("SPAN_KIND_SERVER".to_string()), + ); + } + + #[test] + fn span_status_maps_to_policy_codes() { + assert_eq!( + get( + json!({"status": {"code": "STATUS_CODE_OK"}}), + TraceFieldSelector::SpanStatus + ), + Some("SPAN_STATUS_CODE_OK".to_string()), + ); + assert_eq!( + get( + json!({"status": {"code": "STATUS_CODE_ERROR"}}), + TraceFieldSelector::SpanStatus + ), + Some("SPAN_STATUS_CODE_ERROR".to_string()), + ); + assert_eq!( + get( + json!({"status": {"code": "STATUS_CODE_UNSET"}}), + TraceFieldSelector::SpanStatus + ), + Some("SPAN_STATUS_CODE_UNSPECIFIED".to_string()), + ); + } + + #[test] + fn empty_status_is_unset_not_absent() { + // Regression guard: proto3 omits the default STATUS_CODE_UNSET, so a + // present-but-empty `status` must resolve to UNSPECIFIED and `exists`. + assert_eq!( + get(json!({"status": {}}), TraceFieldSelector::SpanStatus), + Some("SPAN_STATUS_CODE_UNSPECIFIED".to_string()), + ); + assert!(exists(json!({"status": {}}), &TraceFieldSelector::SpanStatus)); + } + + #[test] + fn absent_status_is_none() { + assert_eq!( + get(json!({"name": "x"}), TraceFieldSelector::SpanStatus), + None, + ); + assert!(!exists( + json!({"name": "x"}), + &TraceFieldSelector::SpanStatus + )); + } + + #[test] + fn event_name_is_first_non_empty() { + assert_eq!( + get( + json!({"events": [{"name": ""}, {"name": "exception"}, {"name": "other"}]}), + TraceFieldSelector::EventName, + ), + Some("exception".to_string()), + ); + assert_eq!( + get(json!({"events": []}), TraceFieldSelector::EventName), + None, + ); + } + + #[test] + fn span_resource_scope_attributes() { + let mut span = + val(json!({"attributes": [{"key": "http.method", "value": {"stringValue": "GET"}}]})); + let resource = + val(json!({"attributes": [{"key": "service.name", "value": {"stringValue": "api"}}]})); + let scope = val(json!({"attributes": [{"key": "lib", "value": {"stringValue": "x"}}]})); + let adapter = TraceAdapter { + span: &mut span, + resource: Some(&resource), + scope: Some(&scope), + resource_schema_url: None, + scope_schema_url: None, + }; + let g = |sel| adapter.get_field(&sel).map(Cow::into_owned); + assert_eq!( + g(TraceFieldSelector::SpanAttribute(vec![ + "http.method".to_string() + ])), + Some("GET".to_string()), + ); + assert_eq!( + g(TraceFieldSelector::ResourceAttribute(vec![ + "service.name".to_string() + ])), + Some("api".to_string()), + ); + assert_eq!( + g(TraceFieldSelector::ScopeAttribute(vec!["lib".to_string()])), + Some("x".to_string()), + ); + } + + #[test] + fn sampling_threshold_writes_tracestate() { + let mut span = val(json!({"name": "x"})); + { + let mut adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.set_field(&TraceFieldSelector::SamplingThreshold, "0"); + } + assert_eq!( + span.get("traceState").and_then(|v| v.as_str()).as_deref(), + Some("ot=th:0"), + ); + } + + #[test] + fn sampling_threshold_merges_existing_tracestate() { + let mut span = val(json!({"traceState": "ot=rv:abc,vendor=1"})); + { + let mut adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.set_field(&TraceFieldSelector::SamplingThreshold, "8"); + } + let ts = span + .get("traceState") + .and_then(|v| v.as_str()) + .unwrap() + .into_owned(); + assert!(ts.contains("rv:abc"), "rv preserved: {ts}"); + assert!(ts.contains("th:8"), "th written: {ts}"); + assert!(ts.contains("vendor=1"), "other vendor preserved: {ts}"); + } + + #[test] + fn merge_ot_tracestate_cases() { + assert_eq!(merge_ot_tracestate("", "th:8"), "ot=th:8"); + assert_eq!(merge_ot_tracestate("ot=rv:abc", "th:8"), "ot=rv:abc;th:8"); + assert_eq!(merge_ot_tracestate("foo=bar", "th:8"), "ot=th:8,foo=bar"); + // An existing `th` is overwritten, not duplicated. + assert_eq!(merge_ot_tracestate("ot=th:4", "th:8"), "ot=th:8"); + } +} diff --git a/src/transforms/policy/tests.rs b/src/transforms/policy/tests.rs index 28f50392c56a1..da6d1bbfe4eea 100644 --- a/src/transforms/policy/tests.rs +++ b/src/transforms/policy/tests.rs @@ -14,7 +14,7 @@ use tempfile::NamedTempFile; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use vector_lib::config::LogNamespace; -use vector_lib::event::{LogEvent, Metric, MetricKind, MetricValue}; +use vector_lib::event::{LogEvent, Metric, MetricKind, MetricValue, TraceEvent}; use crate::event::{Event, Value}; use crate::test_util::components::init_test; @@ -1956,3 +1956,229 @@ async fn otel_mode_metric_event_passes_through_untouched() { drop(tx); topology.stop().await; } + +// ============================================================================= +// OTel mode: OTLP metrics envelopes (`resourceMetrics`). +// ============================================================================= + +const OTEL_POLICY_DROP_METRIC_BY_NAME: &str = r#"{ + "policies": [ + { + "id": "drop-system-load", + "name": "drop-system-load", + "metric": { + "match": [ { "metric_field": "name", "regex": "^system\\.load.*$" } ], + "keep": false + } + } + ] +}"#; + +#[tokio::test] +async fn otel_mode_metrics_envelope_drops_and_prunes() { + let policies = write_policies(OTEL_POLICY_DROP_METRIC_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // scope-a holds only a system.load metric (dropped → scope pruned); + // scope-b holds system.load (dropped) + http.requests (kept). This + // exercises the two-phase keep/prune index bookkeeping across scopes. + let envelope = Event::from_json_value( + json!({ + "resourceMetrics": [{ + "resource": { "attributes": [] }, + "scopeMetrics": [ + { + "scope": { "name": "scope-a" }, + "metrics": [ + { "name": "system.load.1", "gauge": { "dataPoints": [{ "attributes": [] }] } } + ] + }, + { + "scope": { "name": "scope-b" }, + "metrics": [ + { "name": "system.load.5", "gauge": { "dataPoints": [{ "attributes": [] }] } }, + { "name": "http.requests", "sum": { "dataPoints": [{ "attributes": [] }], "aggregationTemporality": "AGGREGATION_TEMPORALITY_DELTA" } } + ] + } + ] + }] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("metrics envelope forwarded"); + let log = received.into_log(); + + let scope_metrics = log + .get("resourceMetrics") + .and_then(|v| v.as_array()) + .and_then(|rm| rm.first()) + .and_then(|rm| rm.get("scopeMetrics")) + .and_then(|v| v.as_array()) + .expect("scopeMetrics present"); + assert_eq!(scope_metrics.len(), 1, "empty scope-a must be pruned"); + assert_eq!( + scope_metrics[0] + .get("scope") + .and_then(|s| s.get("name")) + .and_then(|n| n.as_str()) + .as_deref(), + Some("scope-b"), + ); + let metrics = scope_metrics[0] + .get("metrics") + .and_then(|v| v.as_array()) + .expect("metrics present"); + assert_eq!(metrics.len(), 1, "only the non-matching metric survives"); + assert_eq!( + metrics[0].get("name").and_then(|n| n.as_str()).as_deref(), + Some("http.requests"), + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_metrics_envelope_all_dropped_drops_event() { + let policies = write_policies(OTEL_POLICY_DROP_METRIC_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = Event::from_json_value( + json!({ + "resourceMetrics": [{ + "scopeMetrics": [{ + "metrics": [ + { "name": "system.load.1", "gauge": { "dataPoints": [{ "attributes": [] }] } } + ] + }] + }] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + // The only metric is dropped, so the whole envelope is pruned away. + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// OTel mode: OTLP traces envelopes (`resourceSpans`). +// ============================================================================= + +const OTEL_POLICY_DROP_TRACE_BY_NAME: &str = r#"{ + "policies": [ + { + "id": "drop-basic", + "name": "drop-basic", + "trace": { + "match": [ { "trace_field": "TRACE_FIELD_NAME", "exact": "basic" } ], + "keep": { "percentage": 0.0 } + } + } + ] +}"#; + +const OTEL_POLICY_KEEP_ERROR_SPANS: &str = r#"{ + "policies": [ + { + "id": "keep-error", + "name": "keep-error", + "trace": { + "match": [ { "span_status": "SPAN_STATUS_CODE_ERROR", "exists": true } ], + "keep": { "percentage": 100.0 } + } + } + ] +}"#; + +fn otlp_trace(value: serde_json::Value) -> Event { + Event::Trace(TraceEvent::from(Value::from(value))) +} + +#[tokio::test] +async fn otel_mode_traces_envelope_drops_and_prunes() { + let policies = write_policies(OTEL_POLICY_DROP_TRACE_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_trace(json!({ + "resourceSpans": [{ + "resource": { "attributes": [] }, + "scopeSpans": [{ + "scope": { "name": "s" }, + "spans": [ + { "name": "basic", "spanId": "a", "traceId": "t" }, + { "name": "keep-me", "spanId": "b", "traceId": "t" } + ] + }] + }] + })); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("traces envelope forwarded"); + let spans = received + .as_trace() + .get("resourceSpans") + .and_then(|v| v.as_array()) + .and_then(|rs| rs.first()) + .and_then(|rs| rs.get("scopeSpans")) + .and_then(|v| v.as_array()) + .and_then(|ss| ss.first()) + .and_then(|ss| ss.get("spans")) + .and_then(|v| v.as_array()) + .expect("spans present"); + assert_eq!(spans.len(), 1, "the matching span is dropped"); + assert_eq!( + spans[0].get("name").and_then(|n| n.as_str()).as_deref(), + Some("keep-me"), + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_traces_sampling_writes_tracestate() { + let policies = write_policies(OTEL_POLICY_KEEP_ERROR_SPANS); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_trace(json!({ + "resourceSpans": [{ + "scopeSpans": [{ + "spans": [ + { "name": "checkout", "spanId": "a", "traceId": "t", "status": { "code": "STATUS_CODE_ERROR" } } + ] + }] + }] + })); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("traces envelope forwarded"); + let span = received + .as_trace() + .get("resourceSpans") + .and_then(|v| v.as_array()) + .and_then(|rs| rs.first()) + .and_then(|rs| rs.get("scopeSpans")) + .and_then(|v| v.as_array()) + .and_then(|ss| ss.first()) + .and_then(|ss| ss.get("spans")) + .and_then(|v| v.as_array()) + .and_then(|s| s.first()) + .cloned() + .expect("span kept"); + // 100% sampling writes the OTel threshold `th=0` into the tracestate. + assert_eq!( + span.get("traceState").and_then(|v| v.as_str()).as_deref(), + Some("ot=th:0"), + ); + + drop(tx); + topology.stop().await; +} diff --git a/src/transforms/policy/transform.rs b/src/transforms/policy/transform.rs index 109b70b1ba615..d83aa038603ae 100644 --- a/src/transforms/policy/transform.rs +++ b/src/transforms/policy/transform.rs @@ -129,15 +129,15 @@ async fn evaluate_flat( | Ok(EvaluateResult::Sample { keep: true, .. }) | Ok(EvaluateResult::RateLimit { allowed: true, .. }) => Some(log), Ok(EvaluateResult::Drop { .. }) => { - emit_dropped(DropReason::PolicyDrop); + emit_dropped(DropReason::PolicyDrop, 1); None } Ok(EvaluateResult::Sample { keep: false, .. }) => { - emit_dropped(DropReason::SampleRejected); + emit_dropped(DropReason::SampleRejected, 1); None } Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { - emit_dropped(DropReason::RateLimited); + emit_dropped(DropReason::RateLimited, 1); None } Err(error) => { From da4dec16c0b0b46159e8c1289537977b09e993cb Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Fri, 29 May 2026 10:52:07 -0400 Subject: [PATCH 7/9] more improvements --- lib/codecs/src/common/otlp.rs | 2 +- src/transforms/policy/adapter.rs | 2 +- src/transforms/policy/config.rs | 14 +++ src/transforms/policy/internal_events.rs | 35 +++++++- src/transforms/policy/otlp_adapter.rs | 89 +++++++++++++------- src/transforms/policy/otlp_metric_adapter.rs | 21 ++--- src/transforms/policy/otlp_trace_adapter.rs | 36 ++++---- src/transforms/policy/tests.rs | 85 +++++++++++++++++++ 8 files changed, 220 insertions(+), 64 deletions(-) diff --git a/lib/codecs/src/common/otlp.rs b/lib/codecs/src/common/otlp.rs index 1f9d21c1f7b37..e1a9491328088 100644 --- a/lib/codecs/src/common/otlp.rs +++ b/lib/codecs/src/common/otlp.rs @@ -71,7 +71,7 @@ fn hex_encode(bytes: &[u8]) -> Vec { } fn hex_decode(bytes: &[u8]) -> Option> { - if bytes.len() % 2 != 0 { + if !bytes.len().is_multiple_of(2) { return None; } let nibble = |c: u8| -> Option { diff --git a/src/transforms/policy/adapter.rs b/src/transforms/policy/adapter.rs index 15f85bb01e5e0..4fb1d364ca6bb 100644 --- a/src/transforms/policy/adapter.rs +++ b/src/transforms/policy/adapter.rs @@ -66,7 +66,7 @@ fn value_path<'m>( } } -fn simple_value_path(mapping: &FieldMapping, field: LogField) -> Option<&OwnedValuePath> { +const fn simple_value_path(mapping: &FieldMapping, field: LogField) -> Option<&OwnedValuePath> { Some(match field { LogField::Body => &mapping.body.0, LogField::SeverityText => &mapping.severity_text.0, diff --git a/src/transforms/policy/config.rs b/src/transforms/policy/config.rs index 63f5afe056a78..4e87ba0116ddc 100644 --- a/src/transforms/policy/config.rs +++ b/src/transforms/policy/config.rs @@ -74,6 +74,20 @@ pub struct PolicyConfig { } /// Iteration mode for the `policy` transform. +/// +/// NOTE: the two modes intentionally use different matching semantics, because +/// they target different data models — `flat` wraps Vector's schema-less +/// `LogEvent`, while `otel` follows the OTLP/JSON spec exactly (matching the +/// `policy-rs` conformance suite). Two differences are worth knowing when +/// moving a policy between modes: +/// +/// * Non-string values: in `flat` mode an integer/float/boolean attribute is +/// stringified and is therefore matchable (and redactable); in `otel` mode +/// only a `stringValue` is matchable — other `AnyValue` variants satisfy +/// `exists` but never a regex/exact match. +/// * Empty values: `otel` mode treats an empty string / valueless `AnyValue` +/// as absent (a `""` body does not `exist`); `flat` mode treats any present +/// value as present. #[configurable_component] #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] diff --git a/src/transforms/policy/internal_events.rs b/src/transforms/policy/internal_events.rs index c460e8a01d4bf..d547e43fd4c1d 100644 --- a/src/transforms/policy/internal_events.rs +++ b/src/transforms/policy/internal_events.rs @@ -42,7 +42,7 @@ pub(crate) struct DropCounts { } impl DropCounts { - pub(crate) fn record(&mut self, reason: DropReason) { + pub(crate) const fn record(&mut self, reason: DropReason) { match reason { DropReason::PolicyDrop => self.policy_drop += 1, DropReason::SampleRejected => self.sample_rejected += 1, @@ -69,6 +69,39 @@ pub(crate) fn emit_dropped(reason: DropReason, count: u64) { }); } +/// Per-envelope accumulator for fail-open evaluation errors. +/// +/// The transform fails open — an evaluation error passes the record through +/// untouched rather than dropping it. Logging one `error!` per record would +/// spam under a systematic failure (e.g. a single malformed input replayed +/// across an envelope), so we record the first error and a count and emit one +/// line per envelope. +#[derive(Default)] +pub(crate) struct EvalErrors { + count: u64, + first: Option, +} + +impl EvalErrors { + pub(crate) fn record(&mut self, error: &dyn std::fmt::Display) { + if self.first.is_none() { + self.first = Some(error.to_string()); + } + self.count += 1; + } + + pub(crate) fn emit(&self) { + if self.count == 0 { + return; + } + error!( + message = "Policy evaluation failed; affected OTLP records were passed through unchanged.", + count = self.count, + error = self.first.as_deref().unwrap_or_default(), + ); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/transforms/policy/otlp_adapter.rs b/src/transforms/policy/otlp_adapter.rs index ab6c35e56cf2a..52c0fe523fa25 100644 --- a/src/transforms/policy/otlp_adapter.rs +++ b/src/transforms/policy/otlp_adapter.rs @@ -38,7 +38,7 @@ use policy_rs::{ }; use vector_lib::event::{LogEvent, ObjectMap, Value}; -use super::internal_events::{DropCounts, DropReason}; +use super::internal_events::{DropCounts, DropReason, EvalErrors}; /// Iterate every record inside an OTLP envelope event, applying policies /// per-record. Returns `Some(log)` to forward (with mutated and possibly @@ -60,15 +60,14 @@ pub(super) async fn evaluate_envelope( }; let mut drops = DropCounts::default(); + let mut errors = EvalErrors::default(); let mut i = 0; while i < resource_logs.len() { - // Lift `resource` out of the entry so the adapter can hold a mutable - // borrow of it alongside the mutable borrow of `scopeLogs` we take - // below (the borrow checker can't prove those paths are disjoint). - // It is re-inserted after all records under this resource are done. - let mut resource = resource_logs[i] - .as_object_mut() - .and_then(|o| o.remove("resource")); + // Lift `resource`/`scope` out (a move, not a clone) so the adapter can + // mutate them alongside the mutable borrow of the records array (the + // borrow checker can't prove those sibling paths are disjoint). They + // are re-attached after the records under them are processed. + let mut resource = lift_child(&mut resource_logs[i], "resource"); let resource_schema_url = resource_logs[i].get("schemaUrl").cloned(); let mut prune_this_rl = false; @@ -79,10 +78,7 @@ pub(super) async fn evaluate_envelope( { let mut j = 0; while j < scope_logs.len() { - // Same trick for `scope`. - let mut scope = scope_logs[j] - .as_object_mut() - .and_then(|o| o.remove("scope")); + let mut scope = lift_child(&mut scope_logs[j], "scope"); let scope_schema_url = scope_logs[j].get("schemaUrl").cloned(); let mut prune_this_sl = false; @@ -94,13 +90,13 @@ pub(super) async fn evaluate_envelope( let mut k = 0; while k < records.len() { let result = { - let mut adapter = OtlpLogAdapter::new( - &mut records[k], - resource.as_mut(), - scope.as_mut(), - resource_schema_url.as_ref(), - scope_schema_url.as_ref(), - ); + let mut adapter = OtlpLogAdapter { + log_record: &mut records[k], + resource: resource.as_mut(), + scope: scope.as_mut(), + resource_schema_url: resource_schema_url.as_ref(), + scope_schema_url: scope_schema_url.as_ref(), + }; engine.evaluate_and_transform(snapshot, &mut adapter).await }; @@ -124,10 +120,7 @@ pub(super) async fn evaluate_envelope( drops.record(DropReason::RateLimited); } Err(error) => { - error!( - message = "Policy evaluation failed; OTLP record passed through unchanged.", - %error, - ); + errors.record(&error); k += 1; } } @@ -135,11 +128,12 @@ pub(super) async fn evaluate_envelope( prune_this_sl = records.is_empty(); } - // Re-attach the (possibly mutated) scope. - if let Some(scope) = scope - && let Some(obj) = scope_logs[j].as_object_mut() + // Re-attach scope only if the entry survives (otherwise it's + // about to be pruned, so re-inserting would be wasted work). + if !prune_this_sl + && let Some(scope) = scope { - obj.insert("scope".into(), scope); + reattach_child(&mut scope_logs[j], "scope", scope); } if prune_this_sl { @@ -151,11 +145,10 @@ pub(super) async fn evaluate_envelope( prune_this_rl = scope_logs.is_empty(); } - // Re-attach the (possibly mutated) resource. - if let Some(resource) = resource - && let Some(obj) = resource_logs[i].as_object_mut() + if !prune_this_rl + && let Some(resource) = resource { - obj.insert("resource".into(), resource); + reattach_child(&mut resource_logs[i], "resource", resource); } if prune_this_rl { @@ -166,6 +159,7 @@ pub(super) async fn evaluate_envelope( } drops.emit(); + errors.emit(); if resource_logs.is_empty() { None @@ -185,6 +179,10 @@ pub(super) struct OtlpLogAdapter<'a> { } impl<'a> OtlpLogAdapter<'a> { + /// Test-only positional constructor. Production code builds the adapter + /// with a struct literal (see `evaluate_envelope`) so the four same-typed + /// optional borrows can't be transposed silently. + #[cfg(test)] pub(super) fn new( log_record: &'a mut Value, resource: Option<&'a mut Value>, @@ -621,6 +619,35 @@ fn insert_simple_string(record: &mut Value, key: &str, value: &str) { } } +// ============================================================================= +// Envelope-iteration helpers shared by the log / metric / trace adapters. +// ============================================================================= + +/// Remove an object child by key and return it (a move, not a clone). Used to +/// lift `resource` / `scope` out of an envelope entry so they can be borrowed +/// alongside a mutable borrow of a sibling array. +pub(super) fn lift_child(entry: &mut Value, key: &str) -> Option { + entry.as_object_mut().and_then(|o| o.remove(key)) +} + +/// Re-attach a previously [`lift_child`]ed value under `key`. +pub(super) fn reattach_child(entry: &mut Value, key: &str, child: Value) { + if let Some(obj) = entry.as_object_mut() { + obj.insert(key.into(), child); + } +} + +/// Whether `entry`'s array field at `key` is empty or absent — i.e. the entry +/// should be pruned from its parent. +pub(super) fn array_field_is_empty(entry: &Value, key: &str) -> bool { + entry + .as_object() + .and_then(|o| o.get(key)) + .and_then(Value::as_array) + .map(|a| a.is_empty()) + .unwrap_or(true) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/transforms/policy/otlp_metric_adapter.rs b/src/transforms/policy/otlp_metric_adapter.rs index 04b4229e2313b..1ceea78d94a09 100644 --- a/src/transforms/policy/otlp_metric_adapter.rs +++ b/src/transforms/policy/otlp_metric_adapter.rs @@ -22,7 +22,9 @@ use policy_rs::{ use vector_lib::event::{LogEvent, Value}; use super::internal_events::{DropReason, emit_dropped}; -use super::otlp_adapter::{attribute_exists_path, find_attribute_path, non_empty}; +use super::otlp_adapter::{ + array_field_is_empty, attribute_exists_path, find_attribute_path, non_empty, +}; /// Data-variant keys of an OTLP `Metric` (proto3 JSON), paired with the /// `MetricFieldSelector::Type` string the engine matches against. @@ -94,12 +96,15 @@ pub(super) async fn evaluate_metrics_envelope( .get_mut("scopeMetrics") .and_then(Value::as_array_mut) { - // `scope_idx` indexes `keep` by original scope position; `j` tracks - // the live position as emptied scopes are removed. + // Nothing mutates `scopeMetrics`/`metrics` between phases, so the + // decision arrays line up with the live arrays. Index defensively + // anyway (`.get(..)` + fail-open fallback) so a future desync + // degrades to "keep" instead of panicking the transform task. + debug_assert_eq!(keep.len(), scope_metrics.len()); let mut scope_idx = 0; let mut j = 0; while j < scope_metrics.len() { - let scope_keep = &keep[scope_idx]; + let scope_keep: &[bool] = keep.get(scope_idx).map_or(&[], Vec::as_slice); scope_idx += 1; if let Some(metrics) = scope_metrics[j] .get_mut("metrics") @@ -107,7 +112,7 @@ pub(super) async fn evaluate_metrics_envelope( { let mut m = 0; metrics.retain(|_| { - let k = scope_keep[m]; + let k = scope_keep.get(m).copied().unwrap_or(true); m += 1; k }); @@ -120,11 +125,7 @@ pub(super) async fn evaluate_metrics_envelope( } } - let prune_rm = resource_metrics[i] - .get("scopeMetrics") - .and_then(Value::as_array) - .map(<[Value]>::is_empty) - .unwrap_or(true); + let prune_rm = array_field_is_empty(&resource_metrics[i], "scopeMetrics"); if prune_rm { resource_metrics.remove(i); } else { diff --git a/src/transforms/policy/otlp_trace_adapter.rs b/src/transforms/policy/otlp_trace_adapter.rs index 72dccff46634a..d85c2c0429ff7 100644 --- a/src/transforms/policy/otlp_trace_adapter.rs +++ b/src/transforms/policy/otlp_trace_adapter.rs @@ -20,8 +20,10 @@ use policy_rs::{ }; use vector_lib::event::{TraceEvent, Value}; -use super::internal_events::{DropCounts, DropReason}; -use super::otlp_adapter::{attribute_exists_path, find_attribute_path, non_empty}; +use super::internal_events::{DropCounts, DropReason, EvalErrors}; +use super::otlp_adapter::{ + attribute_exists_path, find_attribute_path, lift_child, non_empty, reattach_child, +}; /// Iterate every span in an OTLP traces envelope, sampling/dropping in place. /// Returns `true` if any span survives (forward the event), `false` if the @@ -41,13 +43,12 @@ pub(super) async fn evaluate_traces_envelope( }; let mut drops = DropCounts::default(); + let mut errors = EvalErrors::default(); let mut i = 0; while i < resource_spans.len() { // Lift resource/scope out (a move, not a clone) so the adapter can read // them while we mutate the sibling `spans` array; re-attached after. - let resource = resource_spans[i] - .as_object_mut() - .and_then(|o| o.remove("resource")); + let resource = lift_child(&mut resource_spans[i], "resource"); let resource_schema_url = resource_spans[i].get("schemaUrl").cloned(); let mut prune_rs = false; @@ -58,9 +59,7 @@ pub(super) async fn evaluate_traces_envelope( { let mut j = 0; while j < scope_spans.len() { - let scope = scope_spans[j] - .as_object_mut() - .and_then(|o| o.remove("scope")); + let scope = lift_child(&mut scope_spans[j], "scope"); let scope_schema_url = scope_spans[j].get("schemaUrl").cloned(); let mut prune_ss = false; @@ -93,10 +92,7 @@ pub(super) async fn evaluate_traces_envelope( } Ok(_) => true, Err(error) => { - error!( - message = "Policy evaluation failed; OTLP span passed through unchanged.", - %error, - ); + errors.record(&error); true } }; @@ -110,11 +106,11 @@ pub(super) async fn evaluate_traces_envelope( prune_ss = spans.is_empty(); } - // Re-attach the (read-only) scope. - if let Some(scope) = scope - && let Some(obj) = scope_spans[j].as_object_mut() + // Re-attach scope only if the entry survives. + if !prune_ss + && let Some(scope) = scope { - obj.insert("scope".into(), scope); + reattach_child(&mut scope_spans[j], "scope", scope); } if prune_ss { @@ -126,11 +122,10 @@ pub(super) async fn evaluate_traces_envelope( prune_rs = scope_spans.is_empty(); } - // Re-attach the (read-only) resource. - if let Some(resource) = resource - && let Some(obj) = resource_spans[i].as_object_mut() + if !prune_rs + && let Some(resource) = resource { - obj.insert("resource".into(), resource); + reattach_child(&mut resource_spans[i], "resource", resource); } if prune_rs { @@ -141,6 +136,7 @@ pub(super) async fn evaluate_traces_envelope( } drops.emit(); + errors.emit(); !resource_spans.is_empty() } diff --git a/src/transforms/policy/tests.rs b/src/transforms/policy/tests.rs index da6d1bbfe4eea..87bff4c9359b1 100644 --- a/src/transforms/policy/tests.rs +++ b/src/transforms/policy/tests.rs @@ -2182,3 +2182,88 @@ async fn otel_mode_traces_sampling_writes_tracestate() { drop(tx); topology.stop().await; } + +// --- Envelope edge cases (guard the two-phase / lift hardening) ------------- + +#[tokio::test] +async fn otel_mode_metrics_scope_without_metrics_key_passes_through() { + let policies = write_policies(OTEL_POLICY_DROP_METRIC_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // One scope has no `metrics` array at all (empty decision row in the + // two-phase pass); it must survive untouched rather than panic or prune. + let envelope = Event::from_json_value( + json!({ + "resourceMetrics": [{ + "scopeMetrics": [ + { "scope": { "name": "no-metrics" } }, + { "scope": { "name": "has-metrics" }, "metrics": [ + { "name": "http.requests", "gauge": { "dataPoints": [{ "attributes": [] }] } } + ]} + ] + }] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + let scope_metrics = log + .get("resourceMetrics") + .and_then(|v| v.as_array()) + .and_then(|rm| rm.first()) + .and_then(|rm| rm.get("scopeMetrics")) + .and_then(|v| v.as_array()) + .expect("scopeMetrics present"); + assert_eq!(scope_metrics.len(), 2, "the metrics-less scope must survive"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_traces_without_resource_or_scope() { + let policies = write_policies(OTEL_POLICY_DROP_TRACE_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // No `resource` / `scope` objects: `lift_child` returns `None`. A kept span + // must survive, and the absent resource must not be synthesized on output. + let envelope = otlp_trace(json!({ + "resourceSpans": [{ + "scopeSpans": [{ + "spans": [ { "name": "keep-me", "spanId": "a", "traceId": "t" } ] + }] + }] + })); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("trace forwarded"); + let first_rs = received + .as_trace() + .get("resourceSpans") + .and_then(|v| v.as_array()) + .and_then(|rs| rs.first()) + .cloned() + .expect("resourceSpans present"); + assert!( + first_rs.get("resource").is_none(), + "absent resource must not be synthesized", + ); + let spans = first_rs + .get("scopeSpans") + .and_then(|v| v.as_array()) + .and_then(|ss| ss.first()) + .and_then(|ss| ss.get("spans")) + .and_then(|v| v.as_array()) + .expect("spans present"); + assert_eq!(spans.len(), 1); + assert_eq!( + spans[0].get("name").and_then(|n| n.as_str()).as_deref(), + Some("keep-me"), + ); + + drop(tx); + topology.stop().await; +} From 1b2e3a1ca9f12178c4fecdf1ed520dc61f1c4969 Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Fri, 29 May 2026 12:31:29 -0400 Subject: [PATCH 8/9] some renames --- src/transforms/policy/mod.rs | 3 +- src/transforms/policy/otlp_common.rs | 269 ++++++++++++++++++ .../{otlp_adapter.rs => otlp_log_adapter.rs} | 151 +--------- src/transforms/policy/otlp_metric_adapter.rs | 2 +- src/transforms/policy/otlp_trace_adapter.rs | 2 +- src/transforms/policy/transform.rs | 4 +- 6 files changed, 282 insertions(+), 149 deletions(-) create mode 100644 src/transforms/policy/otlp_common.rs rename src/transforms/policy/{otlp_adapter.rs => otlp_log_adapter.rs} (85%) diff --git a/src/transforms/policy/mod.rs b/src/transforms/policy/mod.rs index 9969d9ed806ff..5c0326235beb5 100644 --- a/src/transforms/policy/mod.rs +++ b/src/transforms/policy/mod.rs @@ -9,7 +9,8 @@ mod adapter; mod field_mapping; mod internal_events; -mod otlp_adapter; +mod otlp_common; +mod otlp_log_adapter; mod otlp_metric_adapter; mod otlp_trace_adapter; diff --git a/src/transforms/policy/otlp_common.rs b/src/transforms/policy/otlp_common.rs new file mode 100644 index 0000000000000..f053147d90261 --- /dev/null +++ b/src/transforms/policy/otlp_common.rs @@ -0,0 +1,269 @@ +//! Shared OTLP value helpers used by the log, metric, and trace adapters. +//! +//! These operate on the JSON-shaped `Value` tree the `opentelemetry` source +//! produces (camelCase keys, `AnyValue`-wrapped attribute values, attributes as +//! `{ key, value }` arrays). They are the read-side primitives — attribute +//! lookup, `AnyValue` coercion, and the envelope lift/prune helpers — that all +//! three signal adapters need; per-signal matching/transform logic lives in the +//! respective `otlp_*_adapter` module. + +use std::borrow::Cow; + +use vector_lib::event::Value; + +// ============================================================================= +// AnyValue coercion. +// ============================================================================= + +/// Coerce a plain string `Value` to a non-empty string. Empty strings count +/// as absent, matching the conformance reference adapter's `non_empty` helper. +pub(super) fn non_empty(value: Option<&Value>) -> Option> { + match value?.as_str() { + Some(s) if !s.is_empty() => Some(s), + _ => None, + } +} + +/// Coerce an OTLP `AnyValue` to a string for matching. Only a non-empty +/// `stringValue` is matchable; all other variants (`intValue`, `boolValue`, +/// `arrayValue`, …) return `None` so string matchers and regex redaction +/// never operate on them. +pub(super) fn any_value_string(value: Option<&Value>) -> Option> { + let obj = value?.as_object()?; + match obj.get("stringValue").and_then(Value::as_str) { + Some(s) if !s.is_empty() => Some(s), + _ => None, + } +} + +/// Whether an `AnyValue` carries any value variant at all. Powers +/// `exists: true` matchers for attributes whose value isn't a string. +fn any_value_present(value: Option<&Value>) -> bool { + const VARIANTS: [&str; 7] = [ + "stringValue", + "boolValue", + "intValue", + "doubleValue", + "arrayValue", + "kvlistValue", + "bytesValue", + ]; + match value.and_then(Value::as_object) { + Some(obj) => VARIANTS.iter().any(|k| obj.get(*k).is_some()), + None => false, + } +} + +// ============================================================================= +// Attribute lookup (walks nested kvlistValue). +// ============================================================================= + +/// Resolve an attribute path against an attributes array, walking nested +/// `kvlistValue` entries for multi-segment paths. Returns the leaf's matchable +/// string value, if any. +pub(super) fn find_attribute_path<'a>( + attrs: Option<&'a Value>, + path: &[String], +) -> Option> { + let array = attrs?.as_array()?; + find_in_kvlist(array, path) +} + +fn find_in_kvlist<'a>(attrs: &'a [Value], path: &[String]) -> Option> { + let first = path.first()?; + for kv in attrs { + if !attribute_key_eq(kv, first) { + continue; + } + let value = kv.as_object().and_then(|o| o.get("value")); + if path.len() == 1 { + return any_value_string(value); + } + return nested_values(value).and_then(|nested| find_in_kvlist(nested, &path[1..])); + } + None +} + +/// Whether an attribute path resolves to a present value (any `AnyValue` +/// variant), walking nested `kvlistValue` entries. +pub(super) fn attribute_exists_path(attrs: Option<&Value>, path: &[String]) -> bool { + let Some(array) = attrs.and_then(Value::as_array) else { + return false; + }; + exists_in_kvlist(array, path) +} + +fn exists_in_kvlist(attrs: &[Value], path: &[String]) -> bool { + let Some(first) = path.first() else { + return false; + }; + for kv in attrs { + if !attribute_key_eq(kv, first) { + continue; + } + let value = kv.as_object().and_then(|o| o.get("value")); + if path.len() == 1 { + return any_value_present(value); + } + return nested_values(value) + .map(|nested| exists_in_kvlist(nested, &path[1..])) + .unwrap_or(false); + } + false +} + +/// Unwrap an `AnyValue`'s `kvlistValue.values` array. +fn nested_values(value: Option<&Value>) -> Option<&[Value]> { + value + .and_then(Value::as_object) + .and_then(|o| o.get("kvlistValue")) + .and_then(Value::as_object) + .and_then(|o| o.get("values")) + .and_then(Value::as_array) +} + +/// Whether an attribute entry's `key` equals `key`, comparing raw bytes so we +/// skip UTF-8 validation on every entry of a linear scan (the key is an OTLP +/// `string`, but for an equality test the byte representation is sufficient). +pub(super) fn attribute_key_eq(item: &Value, key: &str) -> bool { + matches!( + item.as_object().and_then(|o| o.get("key")), + Some(Value::Bytes(b)) if b.as_ref() == key.as_bytes() + ) +} + +// ============================================================================= +// Envelope iteration helpers. +// ============================================================================= + +/// Remove an object child by key and return it (a move, not a clone). Used to +/// lift `resource` / `scope` out of an envelope entry so they can be borrowed +/// alongside a mutable borrow of a sibling array. +pub(super) fn lift_child(entry: &mut Value, key: &str) -> Option { + entry.as_object_mut().and_then(|o| o.remove(key)) +} + +/// Re-attach a previously [`lift_child`]ed value under `key`. +pub(super) fn reattach_child(entry: &mut Value, key: &str, child: Value) { + if let Some(obj) = entry.as_object_mut() { + obj.insert(key.into(), child); + } +} + +/// Whether `entry`'s array field at `key` is empty or absent — i.e. the entry +/// should be pruned from its parent. +pub(super) fn array_field_is_empty(entry: &Value, key: &str) -> bool { + entry + .as_object() + .and_then(|o| o.get(key)) + .and_then(Value::as_array) + .map(|a| a.is_empty()) + .unwrap_or(true) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn v(value: serde_json::Value) -> Value { + Value::from(value) + } + + #[test] + fn non_empty_treats_blank_as_absent() { + assert_eq!(non_empty(Some(&v(json!("x")))).as_deref(), Some("x")); + assert_eq!(non_empty(Some(&v(json!("")))), None); + assert_eq!(non_empty(None), None); + } + + #[test] + fn any_value_string_only_matches_non_empty_string() { + assert_eq!( + any_value_string(Some(&v(json!({"stringValue": "hi"})))).as_deref(), + Some("hi"), + ); + assert_eq!(any_value_string(Some(&v(json!({"stringValue": ""})))), None); + assert_eq!(any_value_string(Some(&v(json!({"intValue": "42"})))), None); + assert_eq!(any_value_string(Some(&v(json!({"boolValue": true})))), None); + assert_eq!(any_value_string(None), None); + } + + #[test] + fn find_attribute_path_flat_and_nested() { + let attrs = v(json!([ + {"key": "user_id", "value": {"stringValue": "42"}}, + {"key": "http", "value": {"kvlistValue": {"values": [ + {"key": "method", "value": {"stringValue": "GET"}} + ]}}} + ])); + assert_eq!( + find_attribute_path(Some(&attrs), &["user_id".to_string()]).as_deref(), + Some("42"), + ); + assert_eq!( + find_attribute_path(Some(&attrs), &["http".to_string(), "method".to_string()]) + .as_deref(), + Some("GET"), + ); + // Missing top-level key, and a missing nested segment, both resolve to None. + assert_eq!(find_attribute_path(Some(&attrs), &["nope".to_string()]), None); + assert_eq!( + find_attribute_path(Some(&attrs), &["http".to_string(), "nope".to_string()]), + None, + ); + } + + #[test] + fn find_attribute_path_non_string_leaf_is_none() { + let attrs = v(json!([{"key": "count", "value": {"intValue": "42"}}])); + assert_eq!(find_attribute_path(Some(&attrs), &["count".to_string()]), None); + } + + #[test] + fn attribute_exists_path_covers_non_string_and_nested() { + let attrs = v(json!([ + {"key": "count", "value": {"intValue": "42"}}, + {"key": "http", "value": {"kvlistValue": {"values": [ + {"key": "method", "value": {"stringValue": "GET"}} + ]}}} + ])); + // A non-string value still satisfies `exists`. + assert!(attribute_exists_path(Some(&attrs), &["count".to_string()])); + assert!(attribute_exists_path( + Some(&attrs), + &["http".to_string(), "method".to_string()] + )); + assert!(!attribute_exists_path(Some(&attrs), &["missing".to_string()])); + assert!(!attribute_exists_path(None, &["count".to_string()])); + } + + #[test] + fn attribute_key_eq_compares_key_bytes() { + let kv = v(json!({"key": "service.name", "value": {"stringValue": "api"}})); + assert!(attribute_key_eq(&kv, "service.name")); + assert!(!attribute_key_eq(&kv, "service")); + // Non-object entries never match. + assert!(!attribute_key_eq(&v(json!("x")), "x")); + } + + #[test] + fn lift_and_reattach_round_trip() { + let mut entry = v(json!({"resource": {"attributes": []}, "scopeLogs": []})); + let lifted = lift_child(&mut entry, "resource").expect("resource lifted"); + assert!(entry.as_object().unwrap().get("resource").is_none()); + reattach_child(&mut entry, "resource", lifted); + assert!(entry.as_object().unwrap().get("resource").is_some()); + // Lifting an absent child is a no-op returning None. + assert!(lift_child(&mut entry, "missing").is_none()); + } + + #[test] + fn array_field_is_empty_cases() { + assert!(array_field_is_empty(&v(json!({"a": []})), "a")); + assert!(!array_field_is_empty(&v(json!({"a": [1]})), "a")); + // Absent field, and a present-but-not-array field, both count as empty. + assert!(array_field_is_empty(&v(json!({})), "a")); + assert!(array_field_is_empty(&v(json!({"a": "x"})), "a")); + } +} diff --git a/src/transforms/policy/otlp_adapter.rs b/src/transforms/policy/otlp_log_adapter.rs similarity index 85% rename from src/transforms/policy/otlp_adapter.rs rename to src/transforms/policy/otlp_log_adapter.rs index 52c0fe523fa25..518bd877d38e3 100644 --- a/src/transforms/policy/otlp_adapter.rs +++ b/src/transforms/policy/otlp_log_adapter.rs @@ -39,6 +39,10 @@ use policy_rs::{ use vector_lib::event::{LogEvent, ObjectMap, Value}; use super::internal_events::{DropCounts, DropReason, EvalErrors}; +use super::otlp_common::{ + any_value_string, attribute_exists_path, attribute_key_eq, find_attribute_path, lift_child, + non_empty, reattach_child, +}; /// Iterate every record inside an OTLP envelope event, applying policies /// per-record. Returns `Some(log)` to forward (with mutated and possibly @@ -49,7 +53,7 @@ use super::internal_events::{DropCounts, DropReason, EvalErrors}; /// not an array), the event is forwarded unchanged so users who /// accidentally send non-envelope events through an `otel`-mode transform /// don't lose data silently. -pub(super) async fn evaluate_envelope( +pub(super) async fn evaluate_logs_envelope( engine: &PolicyEngine, snapshot: &PolicySnapshot, mut log: LogEvent, @@ -180,7 +184,7 @@ pub(super) struct OtlpLogAdapter<'a> { impl<'a> OtlpLogAdapter<'a> { /// Test-only positional constructor. Production code builds the adapter - /// with a struct literal (see `evaluate_envelope`) so the four same-typed + /// with a struct literal (see `evaluate_logs_envelope`) so the four same-typed /// optional borrows can't be transposed silently. #[cfg(test)] pub(super) fn new( @@ -400,39 +404,9 @@ impl Transformable for OtlpLogAdapter<'_> { } // ============================================================================= -// AnyValue helpers. +// Log-record `body` presence (log-specific `AnyValue` semantics). // ============================================================================= -/// Coerce an OTLP `AnyValue` to a string for matching. Only a non-empty -/// `stringValue` is matchable; all other variants (`intValue`, `boolValue`, -/// `arrayValue`, …) return `None` so string matchers and regex redaction -/// never operate on them. -fn any_value_string(value: Option<&Value>) -> Option> { - let obj = value?.as_object()?; - match obj.get("stringValue").and_then(Value::as_str) { - Some(s) if !s.is_empty() => Some(s), - _ => None, - } -} - -/// Whether an `AnyValue` carries any value variant at all. Powers -/// `exists: true` matchers for attributes whose value isn't a string. -fn any_value_present(value: Option<&Value>) -> bool { - const VARIANTS: [&str; 7] = [ - "stringValue", - "boolValue", - "intValue", - "doubleValue", - "arrayValue", - "kvlistValue", - "bytesValue", - ]; - match value.and_then(Value::as_object) { - Some(obj) => VARIANTS.iter().any(|k| obj.get(*k).is_some()), - None => false, - } -} - /// Presence semantics for the `body` field: an empty `stringValue` counts as /// missing, but any other present variant counts as present. fn log_body_present(value: Option<&Value>) -> bool { @@ -447,88 +421,6 @@ fn log_body_present(value: Option<&Value>) -> bool { .any(|k| obj.get(*k).is_some()) } -/// Coerce a plain string `Value` to a non-empty string. Empty strings count -/// as absent, matching the reference adapter's `non_empty` helper. -pub(super) fn non_empty(value: Option<&Value>) -> Option> { - match value?.as_str() { - Some(s) if !s.is_empty() => Some(s), - _ => None, - } -} - -// ============================================================================= -// Attribute lookup (walks nested kvlistValue). -// ============================================================================= - -pub(super) fn find_attribute_path<'a>( - attrs: Option<&'a Value>, - path: &[String], -) -> Option> { - let array = attrs?.as_array()?; - find_in_kvlist(array, path) -} - -fn find_in_kvlist<'a>(attrs: &'a [Value], path: &[String]) -> Option> { - let first = path.first()?; - for kv in attrs { - if !attribute_key_eq(kv, first) { - continue; - } - let value = kv.as_object().and_then(|o| o.get("value")); - if path.len() == 1 { - return any_value_string(value); - } - return nested_values(value).and_then(|nested| find_in_kvlist(nested, &path[1..])); - } - None -} - -pub(super) fn attribute_exists_path(attrs: Option<&Value>, path: &[String]) -> bool { - let Some(array) = attrs.and_then(Value::as_array) else { - return false; - }; - exists_in_kvlist(array, path) -} - -fn exists_in_kvlist(attrs: &[Value], path: &[String]) -> bool { - let Some(first) = path.first() else { - return false; - }; - for kv in attrs { - if !attribute_key_eq(kv, first) { - continue; - } - let value = kv.as_object().and_then(|o| o.get("value")); - if path.len() == 1 { - return any_value_present(value); - } - return nested_values(value) - .map(|nested| exists_in_kvlist(nested, &path[1..])) - .unwrap_or(false); - } - false -} - -/// Unwrap an `AnyValue`'s `kvlistValue.values` array. -fn nested_values(value: Option<&Value>) -> Option<&[Value]> { - value - .and_then(Value::as_object) - .and_then(|o| o.get("kvlistValue")) - .and_then(Value::as_object) - .and_then(|o| o.get("values")) - .and_then(Value::as_array) -} - -/// Whether an attribute entry's `key` equals `key`, comparing raw bytes so we -/// skip UTF-8 validation on every entry of a linear scan (the key is an OTLP -/// `string`, but for an equality test the byte representation is sufficient). -fn attribute_key_eq(item: &Value, key: &str) -> bool { - matches!( - item.as_object().and_then(|o| o.get("key")), - Some(Value::Bytes(b)) if b.as_ref() == key.as_bytes() - ) -} - // ============================================================================= // Attribute mutation. // ============================================================================= @@ -619,35 +511,6 @@ fn insert_simple_string(record: &mut Value, key: &str, value: &str) { } } -// ============================================================================= -// Envelope-iteration helpers shared by the log / metric / trace adapters. -// ============================================================================= - -/// Remove an object child by key and return it (a move, not a clone). Used to -/// lift `resource` / `scope` out of an envelope entry so they can be borrowed -/// alongside a mutable borrow of a sibling array. -pub(super) fn lift_child(entry: &mut Value, key: &str) -> Option { - entry.as_object_mut().and_then(|o| o.remove(key)) -} - -/// Re-attach a previously [`lift_child`]ed value under `key`. -pub(super) fn reattach_child(entry: &mut Value, key: &str, child: Value) { - if let Some(obj) = entry.as_object_mut() { - obj.insert(key.into(), child); - } -} - -/// Whether `entry`'s array field at `key` is empty or absent — i.e. the entry -/// should be pruned from its parent. -pub(super) fn array_field_is_empty(entry: &Value, key: &str) -> bool { - entry - .as_object() - .and_then(|o| o.get(key)) - .and_then(Value::as_array) - .map(|a| a.is_empty()) - .unwrap_or(true) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/transforms/policy/otlp_metric_adapter.rs b/src/transforms/policy/otlp_metric_adapter.rs index 1ceea78d94a09..d5bb63e4b2b7c 100644 --- a/src/transforms/policy/otlp_metric_adapter.rs +++ b/src/transforms/policy/otlp_metric_adapter.rs @@ -22,7 +22,7 @@ use policy_rs::{ use vector_lib::event::{LogEvent, Value}; use super::internal_events::{DropReason, emit_dropped}; -use super::otlp_adapter::{ +use super::otlp_common::{ array_field_is_empty, attribute_exists_path, find_attribute_path, non_empty, }; diff --git a/src/transforms/policy/otlp_trace_adapter.rs b/src/transforms/policy/otlp_trace_adapter.rs index d85c2c0429ff7..b65e5371d32da 100644 --- a/src/transforms/policy/otlp_trace_adapter.rs +++ b/src/transforms/policy/otlp_trace_adapter.rs @@ -21,7 +21,7 @@ use policy_rs::{ use vector_lib::event::{TraceEvent, Value}; use super::internal_events::{DropCounts, DropReason, EvalErrors}; -use super::otlp_adapter::{ +use super::otlp_common::{ attribute_exists_path, find_attribute_path, lift_child, non_empty, reattach_child, }; diff --git a/src/transforms/policy/transform.rs b/src/transforms/policy/transform.rs index d83aa038603ae..a245d98965f74 100644 --- a/src/transforms/policy/transform.rs +++ b/src/transforms/policy/transform.rs @@ -14,7 +14,7 @@ use super::adapter::VectorLogAdapter; use super::config::PolicyMode; use super::field_mapping::FieldMapping; use super::internal_events::{DropReason, emit_dropped}; -use super::otlp_adapter::evaluate_envelope; +use super::otlp_log_adapter::evaluate_logs_envelope; use super::otlp_metric_adapter::evaluate_metrics_envelope; use super::otlp_trace_adapter::evaluate_traces_envelope; @@ -80,7 +80,7 @@ impl TaskTransform for Policy { evaluate_metrics_envelope(&engine, &snapshot, log).await } PolicyMode::Otel => { - evaluate_envelope(&engine, &snapshot, log).await + evaluate_logs_envelope(&engine, &snapshot, log).await } }; if let Some(forwarded) = outcome { From 7a88a40085a64977a184c81c6af48f627727776f Mon Sep 17 00:00:00 2001 From: jaronoff97 Date: Fri, 29 May 2026 15:01:40 -0400 Subject: [PATCH 9/9] fix some ci --- LICENSE-3rdparty.csv | 5 ++ deny.toml | 1 + lib/codecs/src/common/otlp.rs | 12 ++-- src/transforms/policy/adapter.rs | 4 +- src/transforms/policy/internal_events.rs | 3 +- src/transforms/policy/otlp_common.rs | 15 ++++- src/transforms/policy/otlp_log_adapter.rs | 62 +++++++++++++------- src/transforms/policy/otlp_metric_adapter.rs | 15 ++++- src/transforms/policy/otlp_trace_adapter.rs | 22 +++---- src/transforms/policy/tests.rs | 6 +- 10 files changed, 98 insertions(+), 47 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index fa77ca0482eb8..c4a7f61cb5bdf 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -280,6 +280,7 @@ fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors +filetime,https://github.com/alexcrichton/filetime,MIT OR Apache-2.0,Alex Crichton finl_unicode,https://github.com/dahosek/finl_unicode,MIT OR Apache-2.0,The finl_unicode Authors flatbuffers,https://github.com/google/flatbuffers,Apache-2.0,"Robert Winslow , FlatBuffers Maintainers" flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " @@ -388,6 +389,7 @@ inotify,https://github.com/hannobraun/inotify,ISC,"Hanno Braun inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley , Joshka" +instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet integer-encoding,https://github.com/dermesser/integer-encoding-rs,MIT,Lewin Bormann inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel @@ -548,6 +550,7 @@ parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " +pbjson,https://github.com/influxdata/pbjson,MIT,Raphael Taylor-Davies pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR Apache-2.0,RustCrypto Developers peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald pem,https://github.com/jcreekmore/pem-rs,MIT,Jonathan Creekmore @@ -567,6 +570,7 @@ piper,https://github.com/notgull/piper,MIT OR Apache-2.0,"Stjepan Glavina +policy-rs,https://github.com/usetero/policy-rs,Apache-2.0,The policy-rs Authors polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " poly1305,https://github.com/RustCrypto/universal-hashes,Apache-2.0 OR MIT,RustCrypto Developers portable-atomic,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic Authors @@ -858,6 +862,7 @@ utf8parse,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Dylan DPC, Hunar Roop Kahlon" uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors +vectorscan-rs-sys,https://github.com/bradlarsen/vectorscan-rs,Apache-2.0 OR MIT,Brad Larsen vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " diff --git a/deny.toml b/deny.toml index 37db54adf7956..c24527ad03c04 100644 --- a/deny.toml +++ b/deny.toml @@ -51,4 +51,5 @@ ignore = [ { id = "RUSTSEC-2026-0104", reason = "rustls-webpki 0.102/0.101 CRL parsing panic - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179); 0.103 already patched to 0.103.13" }, { id = "RUSTSEC-2026-0105", reason = "core2 is unmaintained and all versions yanked - transitive dependency via libflate -> apache-avro, no safe upgrade available" }, { id = "RUSTSEC-2026-0119", reason = "hickory-proto 0.24 - unpatched crate (https://github.com/mongodb/mongo-rust-driver/pull/1682)" }, + { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained - transitive dependency via policy-rs -> notify 7 -> notify-types, no safe upgrade available upstream" }, ] diff --git a/lib/codecs/src/common/otlp.rs b/lib/codecs/src/common/otlp.rs index e1a9491328088..fc0ef6f757bf5 100644 --- a/lib/codecs/src/common/otlp.rs +++ b/lib/codecs/src/common/otlp.rs @@ -95,12 +95,7 @@ mod tests { use vrl::value::Value; fn obj(pairs: Vec<(&str, Value)>) -> Value { - Value::Object( - pairs - .into_iter() - .map(|(k, v)| (k.into(), v)) - .collect(), - ) + Value::Object(pairs.into_iter().map(|(k, v)| (k.into(), v)).collect()) } #[test] @@ -160,7 +155,10 @@ mod tests { "attributes", Value::Array(vec![obj(vec![ ("key", Value::from("spanId")), - ("value", obj(vec![("stringValue", Value::from("not-an-id"))])), + ( + "value", + obj(vec![("stringValue", Value::from("not-an-id"))]), + ), ])]), )]); let before = tree.clone(); diff --git a/src/transforms/policy/adapter.rs b/src/transforms/policy/adapter.rs index 4fb1d364ca6bb..f2f6d8e3711ba 100644 --- a/src/transforms/policy/adapter.rs +++ b/src/transforms/policy/adapter.rs @@ -108,7 +108,9 @@ impl Transformable for VectorLogAdapter<'_> { let Some(path) = value_path(self.mapping, field) else { return false; }; - self.log.remove((PathPrefix::Event, path.as_ref())).is_some() + self.log + .remove((PathPrefix::Event, path.as_ref())) + .is_some() } fn move_field(&mut self, from: &LogFieldSelector, to: &LogFieldSelector) { diff --git a/src/transforms/policy/internal_events.rs b/src/transforms/policy/internal_events.rs index d547e43fd4c1d..8bf3516fbe147 100644 --- a/src/transforms/policy/internal_events.rs +++ b/src/transforms/policy/internal_events.rs @@ -95,7 +95,8 @@ impl EvalErrors { return; } error!( - message = "Policy evaluation failed; affected OTLP records were passed through unchanged.", + message = + "Policy evaluation failed; affected OTLP records were passed through unchanged.", count = self.count, error = self.first.as_deref().unwrap_or_default(), ); diff --git a/src/transforms/policy/otlp_common.rs b/src/transforms/policy/otlp_common.rs index f053147d90261..a0c5a28998257 100644 --- a/src/transforms/policy/otlp_common.rs +++ b/src/transforms/policy/otlp_common.rs @@ -207,7 +207,10 @@ mod tests { Some("GET"), ); // Missing top-level key, and a missing nested segment, both resolve to None. - assert_eq!(find_attribute_path(Some(&attrs), &["nope".to_string()]), None); + assert_eq!( + find_attribute_path(Some(&attrs), &["nope".to_string()]), + None + ); assert_eq!( find_attribute_path(Some(&attrs), &["http".to_string(), "nope".to_string()]), None, @@ -217,7 +220,10 @@ mod tests { #[test] fn find_attribute_path_non_string_leaf_is_none() { let attrs = v(json!([{"key": "count", "value": {"intValue": "42"}}])); - assert_eq!(find_attribute_path(Some(&attrs), &["count".to_string()]), None); + assert_eq!( + find_attribute_path(Some(&attrs), &["count".to_string()]), + None + ); } #[test] @@ -234,7 +240,10 @@ mod tests { Some(&attrs), &["http".to_string(), "method".to_string()] )); - assert!(!attribute_exists_path(Some(&attrs), &["missing".to_string()])); + assert!(!attribute_exists_path( + Some(&attrs), + &["missing".to_string()] + )); assert!(!attribute_exists_path(None, &["count".to_string()])); } diff --git a/src/transforms/policy/otlp_log_adapter.rs b/src/transforms/policy/otlp_log_adapter.rs index 518bd877d38e3..9915eb2b71276 100644 --- a/src/transforms/policy/otlp_log_adapter.rs +++ b/src/transforms/policy/otlp_log_adapter.rs @@ -134,9 +134,7 @@ pub(super) async fn evaluate_logs_envelope( // Re-attach scope only if the entry survives (otherwise it's // about to be pruned, so re-inserting would be wasted work). - if !prune_this_sl - && let Some(scope) = scope - { + if !prune_this_sl && let Some(scope) = scope { reattach_child(&mut scope_logs[j], "scope", scope); } @@ -149,9 +147,7 @@ pub(super) async fn evaluate_logs_envelope( prune_this_rl = scope_logs.is_empty(); } - if !prune_this_rl - && let Some(resource) = resource - { + if !prune_this_rl && let Some(resource) = resource { reattach_child(&mut resource_logs[i], "resource", resource); } @@ -207,10 +203,9 @@ impl<'a> OtlpLogAdapter<'a> { fn attributes_for(&self, selector: &LogFieldSelector) -> Option<&Value> { match selector { LogFieldSelector::LogAttribute(_) => self.log_record.get("attributes"), - LogFieldSelector::ResourceAttribute(_) => self - .resource - .as_deref() - .and_then(|r| r.get("attributes")), + LogFieldSelector::ResourceAttribute(_) => { + self.resource.as_deref().and_then(|r| r.get("attributes")) + } LogFieldSelector::ScopeAttribute(_) => { self.scope.as_deref().and_then(|s| s.get("attributes")) } @@ -416,9 +411,16 @@ fn log_body_present(value: Option<&Value>) -> bool { if let Some(s) = obj.get("stringValue").and_then(Value::as_str) { return !s.is_empty(); } - ["boolValue", "intValue", "doubleValue", "arrayValue", "kvlistValue", "bytesValue"] - .iter() - .any(|k| obj.get(*k).is_some()) + [ + "boolValue", + "intValue", + "doubleValue", + "arrayValue", + "kvlistValue", + "bytesValue", + ] + .iter() + .any(|k| obj.get(*k).is_some()) } // ============================================================================= @@ -569,7 +571,10 @@ mod tests { let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); // get_field must return None (not "42") so a regex redact won't fire. assert_eq!( - get(&adapter, LogFieldSelector::LogAttribute(vec!["count".into()])), + get( + &adapter, + LogFieldSelector::LogAttribute(vec!["count".into()]) + ), None ); // but exists: true must still match. @@ -619,7 +624,10 @@ mod tests { let mut record = Value::Object(ObjectMap::new()); { let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); - adapter.set_field(&LogFieldSelector::Simple(LogField::Body), "[no body provided]"); + adapter.set_field( + &LogFieldSelector::Simple(LogField::Body), + "[no body provided]", + ); } let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); assert_eq!( @@ -685,8 +693,7 @@ mod tests { ("old", make_string_any_value("val")), ]); { - let mut adapter = - OtlpLogAdapter::new(&mut record, None, Some(&mut scope), None, None); + let mut adapter = OtlpLogAdapter::new(&mut record, None, Some(&mut scope), None, None); assert!(adapter.delete_field(&LogFieldSelector::ScopeAttribute(vec!["secret".into()]))); adapter.move_field( &LogFieldSelector::ScopeAttribute(vec!["old".into()]), @@ -694,10 +701,25 @@ mod tests { ); } let adapter = OtlpLogAdapter::new(&mut record, None, Some(&mut scope), None, None); - assert!(get(&adapter, LogFieldSelector::ScopeAttribute(vec!["secret".into()])).is_none()); - assert!(get(&adapter, LogFieldSelector::ScopeAttribute(vec!["old".into()])).is_none()); + assert!( + get( + &adapter, + LogFieldSelector::ScopeAttribute(vec!["secret".into()]) + ) + .is_none() + ); + assert!( + get( + &adapter, + LogFieldSelector::ScopeAttribute(vec!["old".into()]) + ) + .is_none() + ); assert_eq!( - get(&adapter, LogFieldSelector::ScopeAttribute(vec!["new".into()])), + get( + &adapter, + LogFieldSelector::ScopeAttribute(vec!["new".into()]) + ), Some("val".to_string()) ); } diff --git a/src/transforms/policy/otlp_metric_adapter.rs b/src/transforms/policy/otlp_metric_adapter.rs index d5bb63e4b2b7c..037418c16e163 100644 --- a/src/transforms/policy/otlp_metric_adapter.rs +++ b/src/transforms/policy/otlp_metric_adapter.rs @@ -216,7 +216,10 @@ impl Matchable for MetricAdapter<'_> { MetricFieldSelector::Type => self.data().map(|(_, ty)| Cow::Borrowed(ty)), MetricFieldSelector::Temporality => { let (data, _) = self.data()?; - non_empty(data.as_object().and_then(|o| o.get("aggregationTemporality"))) + non_empty( + data.as_object() + .and_then(|o| o.get("aggregationTemporality")), + ) } } } @@ -269,8 +272,14 @@ mod tests { #[test] fn metric_type_derived_from_data_variant() { let cases = [ - (json!({"name": "m", "gauge": {"dataPoints": []}}), "METRIC_TYPE_GAUGE"), - (json!({"name": "m", "sum": {"dataPoints": []}}), "METRIC_TYPE_SUM"), + ( + json!({"name": "m", "gauge": {"dataPoints": []}}), + "METRIC_TYPE_GAUGE", + ), + ( + json!({"name": "m", "sum": {"dataPoints": []}}), + "METRIC_TYPE_SUM", + ), ( json!({"name": "m", "histogram": {"dataPoints": []}}), "METRIC_TYPE_HISTOGRAM", diff --git a/src/transforms/policy/otlp_trace_adapter.rs b/src/transforms/policy/otlp_trace_adapter.rs index b65e5371d32da..cfd4168b0ead9 100644 --- a/src/transforms/policy/otlp_trace_adapter.rs +++ b/src/transforms/policy/otlp_trace_adapter.rs @@ -107,9 +107,7 @@ pub(super) async fn evaluate_traces_envelope( } // Re-attach scope only if the entry survives. - if !prune_ss - && let Some(scope) = scope - { + if !prune_ss && let Some(scope) = scope { reattach_child(&mut scope_spans[j], "scope", scope); } @@ -122,9 +120,7 @@ pub(super) async fn evaluate_traces_envelope( prune_rs = scope_spans.is_empty(); } - if !prune_rs - && let Some(resource) = resource - { + if !prune_rs && let Some(resource) = resource { reattach_child(&mut resource_spans[i], "resource", resource); } @@ -187,9 +183,7 @@ impl TraceAdapter<'_> { match status.get("code").and_then(Value::as_str).as_deref() { Some("STATUS_CODE_OK") => Some(Cow::Borrowed("SPAN_STATUS_CODE_OK")), Some("STATUS_CODE_ERROR") => Some(Cow::Borrowed("SPAN_STATUS_CODE_ERROR")), - Some("STATUS_CODE_UNSET") | None => { - Some(Cow::Borrowed("SPAN_STATUS_CODE_UNSPECIFIED")) - } + Some("STATUS_CODE_UNSET") | None => Some(Cow::Borrowed("SPAN_STATUS_CODE_UNSPECIFIED")), Some(_) => None, } } @@ -369,7 +363,10 @@ mod tests { Some("GET /x".to_string()), ); assert_eq!( - get(span.clone(), TraceFieldSelector::Simple(TraceField::TraceId)), + get( + span.clone(), + TraceFieldSelector::Simple(TraceField::TraceId) + ), Some("abc".to_string()), ); assert_eq!( @@ -429,7 +426,10 @@ mod tests { get(json!({"status": {}}), TraceFieldSelector::SpanStatus), Some("SPAN_STATUS_CODE_UNSPECIFIED".to_string()), ); - assert!(exists(json!({"status": {}}), &TraceFieldSelector::SpanStatus)); + assert!(exists( + json!({"status": {}}), + &TraceFieldSelector::SpanStatus + )); } #[test] diff --git a/src/transforms/policy/tests.rs b/src/transforms/policy/tests.rs index 87bff4c9359b1..f5d851fa10ce8 100644 --- a/src/transforms/policy/tests.rs +++ b/src/transforms/policy/tests.rs @@ -2217,7 +2217,11 @@ async fn otel_mode_metrics_scope_without_metrics_key_passes_through() { .and_then(|rm| rm.get("scopeMetrics")) .and_then(|v| v.as_array()) .expect("scopeMetrics present"); - assert_eq!(scope_metrics.len(), 2, "the metrics-less scope must survive"); + assert_eq!( + scope_metrics.len(), + 2, + "the metrics-less scope must survive" + ); drop(tx); topology.stop().await;