AE control auth + CodeRabbit #53 followup

.arclint

@@ -20,6 +20,7 @@
     "(^private\/credentials\/.*\\.yaml)",
     "(^src/operator/client/versioned/)",
     "(^src/operator/apis/px.dev/v1alpha1/zz_generated.deepcopy.go)",
+    "(^src/e2e_test/adaptive_export_loadtest/tools/loadgen/)",
     "(^src/stirling/bpf_tools/bcc_bpf/system-headers)",
     "(^src/stirling/mysql/testing/.*\\.json$)",
     "(^src/stirling/obj_tools/testdata/go/test_go_binary.go)",

.bazelignore

@@ -6,3 +6,7 @@ third_party/threadstacks
 tools/chef/nodes
 # To keep third party dependencies separate, privy is intentional setup as a separate bazel workspace
 src/datagen/pii/privy
+
+# adaptive_export_loadtest generator is a docker-built test tool (see its README);
+# build-agent to replace with a bazel target. Until then, keep it out of gazelle.
+src/e2e_test/adaptive_export_loadtest/tools/loadgen

.github/workflows/cli_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read
@@ -209,7 +209,7 @@ jobs:
   update-gh-artifacts-manifest:
     if: |
       always() && needs.create-github-release.result == 'success'
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: [get-dev-image, create-github-release]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}

.github/workflows/cloud_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read

.github/workflows/mirror_demos.yaml

@@ -9,7 +9,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     steps:
     - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v2
       with:

.github/workflows/mirror_deps.yaml

@@ -9,7 +9,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     steps:
     - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v2
       with:

.github/workflows/mirror_releases.yaml

@@ -10,7 +10,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     steps:
     - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v2
       with:

.github/workflows/operator_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read
@@ -140,7 +140,7 @@ jobs:
         git commit -s -m "Release Helm chart ${VERSION}"
         git push origin "gh-pages"
   update-gh-artifacts-manifest:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: [get-dev-image, create-github-release]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}

.github/workflows/perf_common.yaml

@@ -34,7 +34,7 @@ jobs:
       ref: ${{ inputs.ref }}
   generate-perf-matrix:
     needs: get-dev-image-with-extras
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     container:
       image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }}
     outputs:
@@ -57,7 +57,7 @@ jobs:
         echo "matrix=${matrix}" >> $GITHUB_OUTPUT
   run-perf-eval:
     needs: [get-dev-image-with-extras, generate-perf-matrix]
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     container:
       image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }}
     strategy:

.github/workflows/vizier_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read
@@ -140,7 +140,7 @@ jobs:
         git commit -s -m "Release Helm chart Vizier ${VERSION}"
         git push origin "gh-pages"
   update-gh-artifacts-manifest:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: [get-dev-image, create-github-release]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}

bazel/ui.bzl

@@ -17,9 +17,11 @@
 # This file contains rules for for our UI builds.
 
 ui_shared_cmds_start = [
+    "set -x",
     'export BASE_PATH="$(pwd)"',
-    "export PATH=/usr/local/bin:/opt/px_dev/tools/node/bin:$PATH",
-    'export HOME="$(mktemp -d)"',  # This makes node-gyp happy.
+    "export PATH=/opt/px_dev/tools/node/bin:/usr/local/bin:$PATH",
+    "hash -r",
+    'export HOME="$(mktemp -d)"',
     'export TMPPATH="$(mktemp -d)"',
 ]
 
@@ -49,6 +51,7 @@ def _pl_webpack_deps_impl(ctx):
         execution_requirements = {tag: "" for tag in ctx.attr.tags},
         outputs = [out],
         command = " && ".join(cmd),
+        use_default_shell_env = True,
         progress_message =
             "Generating webpack deps %s" % out.short_path,
     )
@@ -72,8 +75,8 @@ def _pl_webpack_library_impl(ctx):
         # and apply it to the environment here. Hopefully,
         # no special characters/spaces/quotes in the results ...
         env_cmds = [
-            '$(sed -E "s/^([A-Za-z_]+)\\s*(.*)/export \\1=\\2/g" "{}")'.format(ctx.info_file.path),
-            '$(sed -E "s/^([A-Za-z_]+)\\s*(.*)/export \\1=\\2/g" "{}")'.format(ctx.version_file.path),
+            '$(sed -E -n "s/^(STABLE_BUILD_TAG|BUILD_TIMESTAMP)\\s+(.*)/export \\1=\\2/p" "{}")'.format(ctx.info_file.path),
+            '$(sed -E -n "s/^(STABLE_BUILD_TAG|BUILD_TIMESTAMP)\\s+(.*)/export \\1=\\2/p" "{}")'.format(ctx.version_file.path),
         ]
         all_files.append(ctx.info_file)
         all_files.append(ctx.version_file)
@@ -95,6 +98,7 @@ def _pl_webpack_library_impl(ctx):
         execution_requirements = {tag: "" for tag in ctx.attr.tags},
         outputs = [out],
         command = " && ".join(cmd),
+        use_default_shell_env = True,
         progress_message =
             "Generating webpack bundle %s" % out.short_path,
     )
@@ -170,6 +174,7 @@ def _pl_deps_licenses_impl(ctx):
         execution_requirements = {tag: "" for tag in ctx.attr.tags},
         outputs = [out],
         command = " && ".join(cmd),
+        use_default_shell_env = True,
         progress_message =
             "Generating licenses %s" % out.short_path,
     )

k8s/vizier/bootstrap/adaptive_export_deployment.yaml

@@ -88,13 +88,24 @@ spec:
         #   value: "changeme-ingest"
         # - name: CLICKHOUSE_DATABASE
         #   value: "forensic_db"
+        # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the
+        # same service-tls-certs secret the broker/PEM use; without this the dx
+        # bearer JWT crosses the CNI in cleartext. Harmless when control is off.
+        volumeMounts:
+        - name: certs
+          mountPath: /certs
+          readOnly: true
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
             - ALL
           seccompProfile:
             type: RuntimeDefault
+      volumes:
+      - name: certs
+        secret:
+          secretName: service-tls-certs
       securityContext:
         runAsUser: 10100
         runAsGroup: 10100

skaffold/skaffold_cloud.yaml

@@ -2,7 +2,6 @@
 .common_bazel_args: &common_bazel_args
 - --compilation_mode=opt
 - --config=stamp
-- --action_env=GOOGLE_APPLICATION_CREDENTIALS
 - --config=x86_64_sysroot
 apiVersion: skaffold/v4beta1
 kind: Config
@@ -77,78 +76,9 @@ manifests:
     paths:
     - k8s/cloud/dev
 profiles:
-- name: minikube
-  activation:
-  - kubeContext: minikube
-  patches:
-  - op: replace
-    path: /build/local
-    value:
-      push: false
-- name: dev
-  activation:
-  - command: dev
-  patches:
-  - op: add
-    path: /build/artifacts/context=./bazel/args
-    value:
-    - --compilation_mode=dbg
-- name: ory_auth
-  patches:
-  - op: replace
-    path: /manifests/kustomize/paths
-    value:
-    - k8s/cloud/dev
-    - k8s/cloud/dev/ory_auth
-  - op: add
-    path: /build/artifacts/context=./bazel/args
-    value:
-    - --compilation_mode=dbg
-- name: ory_auth_prod
-  patches:
-  - op: add
-    path: /manifests/kustomize/paths
-    value:
-    - k8s/cloud/base/ory_auth
-- name: staging
-  activation:
-  - env: PL_BUILD_TYPE=staging
-  patches:
-  - op: add
-    path: /build/artifacts/context=./bazel/args
-    value:
-      *common_bazel_args
-  - op: replace
-    path: /manifests/kustomize/paths
-    value:
-    - k8s/cloud/staging
-- name: testing
-  activation:
-  - env: PL_BUILD_TYPE=testing
-  patches:
-  - op: add
-    path: /build/artifacts/context=./bazel/args
-    value:
-      *common_bazel_args
-  - op: replace
-    path: /manifests/kustomize/paths
-    value:
-    - k8s/cloud/testing
-- name: prod
-  activation:
-  - env: PL_BUILD_TYPE=prod
-  patches:
-  - op: add
-    path: /build/artifacts/context=./bazel/args
-    value:
-      *common_bazel_args
-  - op: replace
-    path: /manifests/kustomize/paths
-    value:
-    - k8s/cloud/prod
-- name: public
+- name: cockpit
   patches:
   - op: replace
     path: /manifests/kustomize/paths
     value:
-    - k8s/cloud/public/base
+    - cockpit

src/api/go/pxapi/opts.go

@@ -82,3 +82,17 @@ func WithDirectCredsInsecure() ClientOption {
 		c.insecureDirect = true
 	}
 }
+
+// WithDirectTLSSkipVerify is the secure-by-default option for direct (standalone /
+// node-local PEM) connections: the transport IS TLS-encrypted, but the server cert
+// is not chain/hostname-verified. Use this instead of WithDirectCredsInsecure when
+// the direct endpoint serves TLS with a self-signed / service cert whose SAN does
+// not match the node IP (e.g. vizier-pem's direct-query port served with
+// service-tls-certs, dialed at HOST_IP). Unlike WithDisableTLSVerification it does
+// NOT require a "cluster.local" address, so it works for the node-IP direct dial.
+// Bearer creds (the minted JWT) therefore ride an encrypted channel, never plaintext.
+func WithDirectTLSSkipVerify() ClientOption {
+	return func(c *Client) {
+		c.disableTLSVerification = true
+	}
+}