From 1cb0dd02a06edb9d735c191062de64e62d03338a Mon Sep 17 00:00:00 2001 From: shobham Date: Fri, 24 Jul 2026 17:37:54 +0530 Subject: [PATCH 1/2] feat(byoo-perf): add scaffolding, translation-driven rendering, and validation Add the BYOO collector performance-test module under src/compute-plane-services/byoo-otel-collector/perf as a standalone Go module. - cmd/perf CLI with render/run/cleanup and dev/baseline profiles - pkg/spec: synthetic container and Helm launch specs + translate config - pkg/render: render via icms-translate function.Translate and extract the authentic collector (container sidecar and Helm utils-pod shapes) - pkg/validate: render-shape gate for placement, image, resources, ports, volumes, environment, probes, and config; fails clearly on translator drift - unit tests for spec, render, and validate The collector spec and config are produced entirely by the shared translation library so the suite exercises the same workload that ships in production. Part of #416. Closes #417. Signed-off-by: shobham --- .../byoo-otel-collector/perf/README.md | 67 +++++ .../byoo-otel-collector/perf/cmd/perf/main.go | 207 ++++++++++++++++ .../byoo-otel-collector/perf/go.mod | 50 ++++ .../byoo-otel-collector/perf/go.sum | 113 +++++++++ .../perf/pkg/profile/profile.go | 80 ++++++ .../perf/pkg/render/render.go | 156 ++++++++++++ .../perf/pkg/render/render_test.go | 88 +++++++ .../byoo-otel-collector/perf/pkg/spec/spec.go | 228 +++++++++++++++++ .../perf/pkg/spec/spec_test.go | 105 ++++++++ .../perf/pkg/validate/validate.go | 232 ++++++++++++++++++ .../perf/pkg/validate/validate_test.go | 94 +++++++ 11 files changed, 1420 insertions(+) create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/README.md create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/cmd/perf/main.go create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/go.mod create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/go.sum create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/pkg/profile/profile.go create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render.go create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render_test.go create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec.go create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec_test.go create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate.go create mode 100644 src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate_test.go diff --git a/src/compute-plane-services/byoo-otel-collector/perf/README.md b/src/compute-plane-services/byoo-otel-collector/perf/README.md new file mode 100644 index 000000000..bccb31676 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/README.md @@ -0,0 +1,67 @@ +# BYOO Collector Performance Test Suite + +A repeatable performance test suite for the BYOO OpenTelemetry collector. It +exercises the collector under controlled telemetry load using the same workload +shape produced in production, by rendering through the shared translation +library (`icms-translate`) rather than hand-written collector manifests. + +> Status: first milestone. Only rendering and render-shape validation are +> implemented today. Deployment, load generation, the OTLP sink, measurements, +> and reporting land in later milestones. + +## Why translation-driven + +The collector spec and config are generated by calling +`function.Translate(msg, tcfg)` from the shared translation library, so the +suite tests the identical collector spec and config that would be deployed: + +- Container functions get the collector as a workload sidecar. +- Helm functions get the collector on the generated utils pod, fronted by a + ClusterIP OTLP Service. + +The suite extracts the authentic collector container and, for deployment, runs +it with lightweight stand-ins for the non-collector containers, which are not +what the suite measures. + +## Layout + +- `cmd/perf` — CLI entrypoint (`render`, `run`, `cleanup`). +- `pkg/spec` — synthetic container/Helm launch specs and translate config. +- `pkg/render` — runs `function.Translate` and extracts the collector. +- `pkg/validate` — render-shape validation gate. +- `pkg/profile` — `dev` and `baseline` execution profiles. + +This is a standalone Go module. Run its Go commands with `GOWORK=off`: + +```bash +GOWORK=off go build ./... +GOWORK=off go test ./... +``` + +## Usage + +```bash +# Render and validate both shapes (no cluster required). +GOWORK=off go run ./cmd/perf render --shape both + +# Render one shape and print the deployable workload as YAML. +GOWORK=off go run ./cmd/perf render --shape container --output yaml +``` + +### `render` flags + +- `--shape` (`both`): `container`, `helm`, or `both`. +- `--profile` (`dev`): `dev` or `baseline`. +- `--collector-image`: BYOO collector image reference. +- `--namespace` (`byoo-perf`): namespace for rendered objects. +- `--output` (`summary`): `summary`, `yaml`, or `json`. + +`run` and `cleanup` are scaffolded and return a clear "not implemented" message. + +## Execution profiles + +- `dev`: short run (10s warmup, 30s window, 1 repetition) for wiring and local + iteration. +- `baseline`: longer, repeatable run (60s warmup, 5m window, 3 repetitions). + +All profile values are documented defaults and will be overridable per run. diff --git a/src/compute-plane-services/byoo-otel-collector/perf/cmd/perf/main.go b/src/compute-plane-services/byoo-otel-collector/perf/cmd/perf/main.go new file mode 100644 index 000000000..1fba38354 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/cmd/perf/main.go @@ -0,0 +1,207 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command perf is the entrypoint for the BYOO collector performance test +// suite. In this first milestone only "render" is fully wired; "run" and +// "cleanup" are scaffolding for the deployment/load milestones (S4+). +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/render" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + var err error + switch os.Args[1] { + case "render": + err = cmdRender(os.Args[2:]) + case "run": + err = cmdRun(os.Args[2:]) + case "cleanup": + err = cmdCleanup(os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "unknown command %q\n\n", os.Args[1]) + usage() + os.Exit(2) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `perf - BYOO OpenTelemetry collector performance test suite + +Usage: + perf [flags] + +Commands: + render Render the production workload shape via icms-translate and validate it (no cluster required). + run Deploy, drive load, and measure (not yet implemented; see S4+). + cleanup Remove test resources (not yet implemented; see S5+). + +Run "perf -h" for command flags. +`) +} + +func cmdRender(args []string) error { + fs := flag.NewFlagSet("render", flag.ContinueOnError) + shapeFlag := fs.String("shape", "both", `deployment shape: "container", "helm", or "both"`) + profileFlag := fs.String("profile", "dev", `execution profile: "dev" or "baseline"`) + collectorImage := fs.String("collector-image", spec.DefaultCollectorImage, "BYOO collector image reference") + namespace := fs.String("namespace", "byoo-perf", "namespace for rendered objects") + output := fs.String("output", "summary", `output format: "summary", "yaml", or "json"`) + if err := fs.Parse(args); err != nil { + return err + } + + prof, err := profile.Lookup(*profileFlag) + if err != nil { + return err + } + shapes, err := shapesFromFlag(*shapeFlag) + if err != nil { + return err + } + + opts := spec.DefaultOptions() + opts.Namespace = *namespace + opts.CollectorImage = *collectorImage + + exp := validate.Expectations{ + Image: opts.CollectorImage, + Resources: common.GetDefaultContainerResourcesBYOO(), + } + + fmt.Printf("profile=%s warmup=%s window=%s reps=%d\n\n", prof.Name, prof.Warmup, prof.MeasurementWindow, prof.Repetitions) + + for _, shape := range shapes { + res, err := render.Render(shape, opts) + if err != nil { + return fmt.Errorf("render %s: %w", shape, err) + } + if err := validate.Render(res, exp); err != nil { + return err + } + + switch *output { + case "summary": + printSummary(res) + case "yaml", "json": + if err := printObjects(res, *namespace, *output); err != nil { + return err + } + default: + return fmt.Errorf("unknown output %q (want \"summary\", \"yaml\", or \"json\")", *output) + } + } + return nil +} + +func printSummary(res *render.Result) { + fmt.Printf("[%s] VALID\n", res.Shape) + fmt.Printf(" collector image : %s\n", res.Collector.Image) + fmt.Printf(" config version : %s\n", res.OTelVersion) + fmt.Printf(" owner pod : %s\n", res.OwnerPod) + if res.Service != nil { + fmt.Printf(" otlp service : %s\n", res.Service.Name) + } + fmt.Printf(" ports : %s\n", portSummary(res)) + fmt.Printf(" objects : %d translated\n\n", len(res.Objects)) +} + +func portSummary(res *render.Result) string { + parts := make([]string, 0, len(res.Collector.Ports)) + for _, p := range res.Collector.Ports { + parts = append(parts, fmt.Sprintf("%s:%d", p.Name, p.ContainerPort)) + } + return strings.Join(parts, " ") +} + +func printObjects(res *render.Result, namespace, format string) error { + pod := res.BenchPod(namespace) + out, err := yaml.Marshal(pod) + if err != nil { + return fmt.Errorf("marshal bench pod: %w", err) + } + if format == "json" { + out, err = yaml.YAMLToJSON(out) + if err != nil { + return fmt.Errorf("convert to json: %w", err) + } + } + fmt.Printf("# shape=%s benchmark workload (authentic collector + emptyDir stand-ins)\n", res.Shape) + fmt.Printf("%s\n", out) + return nil +} + +func cmdRun(args []string) error { + fs := flag.NewFlagSet("run", flag.ContinueOnError) + _ = fs.String("shape", "both", `deployment shape: "container", "helm", or "both"`) + _ = fs.String("profile", "dev", `execution profile: "dev" or "baseline"`) + _ = fs.String("mode", "k3d", `deployment mode: "k3d" or "remote"`) + _ = fs.Bool("retain", false, "retain test resources for debugging instead of cleaning up") + if err := fs.Parse(args); err != nil { + return err + } + return fmt.Errorf("`run` is not implemented yet; it lands with the deployment/load milestones (S4-S9)") +} + +func cmdCleanup(args []string) error { + fs := flag.NewFlagSet("cleanup", flag.ContinueOnError) + _ = fs.String("mode", "k3d", `deployment mode: "k3d" or "remote"`) + _ = fs.String("namespace", "byoo-perf", "namespace to clean up") + if err := fs.Parse(args); err != nil { + return err + } + return fmt.Errorf("`cleanup` is not implemented yet; it lands with the deployment milestone (S5)") +} + +func shapesFromFlag(s string) ([]spec.Shape, error) { + switch s { + case "container": + return []spec.Shape{spec.ShapeContainer}, nil + case "helm": + return []spec.Shape{spec.ShapeHelm}, nil + case "both": + return []spec.Shape{spec.ShapeContainer, spec.ShapeHelm}, nil + default: + return nil, fmt.Errorf("unknown shape %q (want \"container\", \"helm\", or \"both\")", s) + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/go.mod b/src/compute-plane-services/byoo-otel-collector/perf/go.mod new file mode 100644 index 000000000..7726b59f0 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/go.mod @@ -0,0 +1,50 @@ +module github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf + +go 1.25.0 + +require ( + github.com/NVIDIA/nvcf/src/libraries/go/lib v0.0.0 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/go-containerregistry v0.21.5 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/text v0.37.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect +) + +// The perf suite consumes the shared SIS translation library directly so that +// it renders exactly the workload shape NVCA produces in production. +replace github.com/NVIDIA/nvcf/src/libraries/go/lib => ../../../libraries/go/lib + +// Mirror the k8s pins used by the shared library. replace directives in +// dependencies are not inherited, so we repeat them here for the main module. +replace ( + k8s.io/api => k8s.io/api v0.34.2 + k8s.io/apimachinery => k8s.io/apimachinery v0.34.2 + k8s.io/client-go => k8s.io/client-go v0.34.2 +) diff --git a/src/compute-plane-services/byoo-otel-collector/perf/go.sum b/src/compute-plane-services/byoo-otel-collector/perf/go.sum new file mode 100644 index 000000000..e60a5b515 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/go.sum @@ -0,0 +1,113 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.5 h1:KTJG9Pn/jC0VdZR6ctV3/jcN+q6/Iqlx0sTVz3ywZlM= +github.com/google/go-containerregistry v0.21.5/go.mod h1:ySvMuiWg+dOsRW0Hw8GYwfMwBlNRTmpYBFJPlkco5zU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= +k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= +k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= +k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile/profile.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile/profile.go new file mode 100644 index 000000000..9ec659028 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile/profile.go @@ -0,0 +1,80 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package profile defines the named execution profiles for the BYOO collector +// performance suite. Profiles bundle the run-shape knobs (warmup, measurement +// window, repetitions, and default load rates). All values are overridable via +// CLI flags; the constants here are documented defaults only. +package profile + +import ( + "fmt" + "time" +) + +// Profile is a named bundle of run parameters. +type Profile struct { + // Name is the profile identifier ("dev" or "baseline"). + Name string + // Warmup is the duration load runs before the measurement window opens. + Warmup time.Duration + // MeasurementWindow is the duration over which measurements are recorded. + MeasurementWindow time.Duration + // Repetitions is how many times the measurement window is repeated. + Repetitions int + // LogRecordsPerSec is the default logs load rate. + LogRecordsPerSec int + // MetricDataPointsPerSec is the default metrics load rate. + MetricDataPointsPerSec int +} + +// Dev is a short run intended to validate wiring and support local iteration. +func Dev() Profile { + return Profile{ + Name: "dev", + Warmup: 10 * time.Second, + MeasurementWindow: 30 * time.Second, + Repetitions: 1, + LogRecordsPerSec: 1000, + MetricDataPointsPerSec: 1000, + } +} + +// Baseline is a longer, repeatable run with warmup, a defined measurement +// window, and repeated measurements. +func Baseline() Profile { + return Profile{ + Name: "baseline", + Warmup: 60 * time.Second, + MeasurementWindow: 5 * time.Minute, + Repetitions: 3, + LogRecordsPerSec: 10000, + MetricDataPointsPerSec: 10000, + } +} + +// Lookup returns the named profile, or an error if the name is unknown. +func Lookup(name string) (Profile, error) { + switch name { + case "dev": + return Dev(), nil + case "baseline": + return Baseline(), nil + default: + return Profile{}, fmt.Errorf("unknown profile %q (want \"dev\" or \"baseline\")", name) + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render.go new file mode 100644 index 000000000..ccd5568cf --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render.go @@ -0,0 +1,156 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package render drives the shared SIS translation library to produce the +// production workload shape, then extracts the authentic BYOO collector for +// the performance suite. Rendering touches no cluster; deployment happens +// later (S5/S6). +package render + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +// CollectorContainerName is the container name the translator assigns the BYOO +// collector in every workload shape. +const CollectorContainerName = common.ByooOTelCollectorPodNameBase + +// Result is the outcome of rendering a workload shape through the translator. +type Result struct { + Shape spec.Shape + Options spec.Options + + // Objects is the full set of translated Kubernetes objects. + Objects []metav1.Object + // Collector is the authentic BYOO collector container, exactly as the + // translator emitted it. + Collector corev1.Container + // OwnerPod is the name of the pod that hosts the collector. + OwnerPod string + // Service is the OTLP ClusterIP Service (Helm shape only; nil otherwise). + Service *corev1.Service + // OTelVersion is the collector config code path ("v1" or "v2") the + // translator selected from the collector image tag. + OTelVersion string +} + +// Render builds the synthetic spec for the shape, runs it through +// function.Translate, and extracts the authentic collector container. +func Render(shape spec.Shape, o spec.Options) (*Result, error) { + msg, err := spec.Message(shape, o) + if err != nil { + return nil, err + } + tcfg := spec.TranslateConfig(shape, o) + + objs, err := function.Translate(msg, tcfg) + if err != nil { + return nil, fmt.Errorf("translate %s function: %w", shape, err) + } + + res := &Result{ + Shape: shape, + Options: o, + Objects: objs, + OTelVersion: common.OTelVersion(o.CollectorImage), + } + + found := false + for _, obj := range objs { + switch t := obj.(type) { + case *corev1.Pod: + for i := range t.Spec.Containers { + if t.Spec.Containers[i].Name == CollectorContainerName { + res.Collector = *t.Spec.Containers[i].DeepCopy() + res.OwnerPod = t.Name + found = true + } + } + case *corev1.Service: + if t.Name == common.ByooOTelCollectorPodNameBase { + res.Service = t.DeepCopy() + } + } + } + if !found { + return nil, fmt.Errorf("no %q container found in translated %s workload (are telemetries enabled?)", CollectorContainerName, shape) + } + return res, nil +} + +// HasContainer reports whether any translated pod contains a container with the +// given name (init containers included). +func (r *Result) HasContainer(name string) bool { + for _, obj := range r.Objects { + pod, ok := obj.(*corev1.Pod) + if !ok { + continue + } + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == name { + return true + } + } + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == name { + return true + } + } + } + return false +} + +// BenchPod produces the deployable performance workload: the authentic +// collector container plus lightweight emptyDir stand-ins for every volume it +// mounts. The non-collector function containers (init, GPU inference, utils, +// ESS) are intentionally omitted since they cannot run on k3d and are not what +// the suite measures. Full deployment orchestration is added in S5/S6. +func (r *Result) BenchPod(namespace string) *corev1.Pod { + pod := &corev1.Pod{} + pod.Name = r.Options.ObjectNameBase + "-collector" + pod.Namespace = namespace + pod.Labels = map[string]string{ + common.K8sAppNameLabelKey: common.ByooOTelCollectorPodNameBase, + "app.kubernetes.io/part-of": "byoo-perf", + common.BYOOMetricsEgressTargetLabelKey: common.BYOOMetricsEgressTargetLabelValue, + } + + collector := *r.Collector.DeepCopy() + pod.Spec.Containers = []corev1.Container{collector} + pod.Spec.RestartPolicy = corev1.RestartPolicyNever + + seen := map[string]bool{} + for _, vm := range collector.VolumeMounts { + if seen[vm.Name] { + continue + } + seen[vm.Name] = true + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: vm.Name, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) + } + return pod +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render_test.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render_test.go new file mode 100644 index 000000000..b94a74bbc --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render_test.go @@ -0,0 +1,88 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package render + +import ( + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +func TestRenderContainerExtractsSidecar(t *testing.T) { + res, err := Render(spec.ShapeContainer, spec.DefaultOptions()) + if err != nil { + t.Fatalf("Render: %v", err) + } + if res.Collector.Name != common.ByooOTelCollectorPodNameBase { + t.Errorf("collector name = %q, want %q", res.Collector.Name, common.ByooOTelCollectorPodNameBase) + } + if res.Collector.Image != spec.DefaultCollectorImage { + t.Errorf("collector image = %q, want %q", res.Collector.Image, spec.DefaultCollectorImage) + } + if res.OTelVersion != "v2" { + t.Errorf("otel version = %q, want v2 for the default (>0.119.4) image", res.OTelVersion) + } + if res.OwnerPod != "0-perf" { + t.Errorf("owner pod = %q, want %q", res.OwnerPod, "0-perf") + } + if !res.HasContainer("inference") { + t.Error("container shape must include an inference container") + } + if res.Service != nil { + t.Error("container shape should not produce an OTLP Service") + } +} + +func TestRenderHelmPlacesOnUtilsPodWithService(t *testing.T) { + res, err := Render(spec.ShapeHelm, spec.DefaultOptions()) + if err != nil { + t.Fatalf("Render: %v", err) + } + if res.OwnerPod != common.UtilsPodName { + t.Errorf("owner pod = %q, want %q", res.OwnerPod, common.UtilsPodName) + } + if res.Service == nil { + t.Fatal("helm shape must produce an OTLP Service") + } + if res.Service.Name != common.ByooOTelCollectorPodNameBase { + t.Errorf("service name = %q, want %q", res.Service.Name, common.ByooOTelCollectorPodNameBase) + } +} + +func TestBenchPodSuppliesVolumesForEveryMount(t *testing.T) { + res, err := Render(spec.ShapeContainer, spec.DefaultOptions()) + if err != nil { + t.Fatalf("Render: %v", err) + } + pod := res.BenchPod("byoo-perf") + if len(pod.Spec.Containers) != 1 { + t.Fatalf("bench pod containers = %d, want 1 (authentic collector only)", len(pod.Spec.Containers)) + } + + vols := map[string]bool{} + for _, v := range pod.Spec.Volumes { + vols[v.Name] = true + } + for _, m := range pod.Spec.Containers[0].VolumeMounts { + if !vols[m.Name] { + t.Errorf("collector mount %q has no backing volume in the bench pod", m.Name) + } + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec.go new file mode 100644 index 000000000..ec5827c9e --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec.go @@ -0,0 +1,228 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package spec builds synthetic NVCF function launch specifications and the +// NVCA-equivalent translate configuration used to drive the shared SIS +// translation library. Feeding these through function.Translate yields the +// exact workload shape NVCA produces in production, including BYOO collector +// placement. +package spec + +import ( + "encoding/base64" + "fmt" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" +) + +// Shape is a supported NVCF function deployment shape. +type Shape string + +const ( + // ShapeContainer is a container function; the BYOO collector is a workload + // sidecar on the inference pod. + ShapeContainer Shape = "container" + // ShapeHelm is a Helm function; the BYOO collector runs on the generated + // utils pod and is fronted by a ClusterIP Service. + ShapeHelm Shape = "helm" +) + +// Default image and identifier values for synthetic specs. The collector image +// tag must be greater than the icms-translate v2 cutoff (0.119.4) so the +// rendered collector uses the runtime-config ("v2") code path that matches +// current production. +const ( + DefaultCollectorImage = "nvcr.io/nvidia/nvcf-core/byoo-otel-collector:0.153.1" + DefaultInferenceImage = "nvcr.io/nvidia/nvcf-perf/stub-inference:latest" + DefaultUtilsImage = "nvcr.io/nvidia/nvcf-core/worker-utils:latest" + DefaultInitImage = "nvcr.io/nvidia/nvcf-core/worker-init:latest" + DefaultHelmChartURL = "https://helm.example.invalid/charts/perf-test-chart-0.1.0.tgz" + + // InstanceTypeLabelKey mirrors the label selector key NVCA uses for + // instance-type node affinity. + InstanceTypeLabelKey = "node.kubernetes.io/instance-type" + + instanceTypeValue = "perf.gpu" +) + +// Options controls the synthetic launch specification and translate config. +type Options struct { + Namespace string + ObjectNameBase string + + CollectorImage string + InferenceImage string + UtilsImage string + InitImage string + + ClusterRegion string + ClusterName string + + // Telemetry destination knobs. These are placeholder endpoints for + // rendering; deployment (S5/S6) overrides them to point at the in-cluster + // OTLP sink. + Protocol string + Provider string + LogsEndpoint string + MetricsEndpoint string + EnableLogs bool + EnableMetrics bool +} + +// DefaultOptions returns a self-consistent set of options for local rendering. +func DefaultOptions() Options { + return Options{ + Namespace: "byoo-perf", + ObjectNameBase: "perf", + CollectorImage: DefaultCollectorImage, + InferenceImage: DefaultInferenceImage, + UtilsImage: DefaultUtilsImage, + InitImage: DefaultInitImage, + ClusterRegion: "perf-local", + ClusterName: "perf", + Protocol: "grpc", + Provider: "GRAFANA_CLOUD", + LogsEndpoint: "https://logs.perf.invalid/otlp", + MetricsEndpoint: "https://metrics.perf.invalid/otlp", + EnableLogs: true, + EnableMetrics: true, + } +} + +func (o Options) telemetries() *common.TelemetriesLaunchSpecification { + if !o.EnableLogs && !o.EnableMetrics { + return nil + } + t := &common.TelemetriesLaunchSpecification{} + if o.EnableLogs { + t.Telemetries.Logs = &common.Telemetry{ + Protocol: o.Protocol, Provider: o.Provider, Endpoint: o.LogsEndpoint, Name: "perf-logs", + } + } + if o.EnableMetrics { + t.Telemetries.Metrics = &common.Telemetry{ + Protocol: o.Protocol, Provider: o.Provider, Endpoint: o.MetricsEndpoint, Name: "perf-metrics", + } + } + return t +} + +// Message builds the synthetic creation-queue message for the given shape. +func Message(shape Shape, o Options) (function.CreationQueueMessage, error) { + switch shape { + case ShapeContainer: + return containerMessage(o), nil + case ShapeHelm: + return helmMessage(o), nil + default: + return function.CreationQueueMessage{}, fmt.Errorf("unknown shape %q (want %q or %q)", shape, ShapeContainer, ShapeHelm) + } +} + +func baseMessage() function.CreationQueueMessage { + return function.CreationQueueMessage{ + CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{ + Action: common.FunctionCreationAction, + RequestID: "perf-request", + MessageBatchID: "perf-batch", + NCAID: "perf-nca", + InstanceCount: 1, + InstanceTypeValue: instanceTypeValue, + GPUType: "PERF", + RequestedGPUCount: 1, + }, + Details: function.Details{ + FunctionID: "perf-function", + FunctionVersionID: "perf-version", + FunctionType: function.FunctionTypeDefault, + }, + } +} + +func containerMessage(o Options) function.CreationQueueMessage { + msg := baseMessage() + msg.LaunchSpecification = &function.LaunchSpecification{ + EnvironmentB64: encodeTextEnv(map[string]string{ + common.ContainerFunctionImageEnv: o.InferenceImage, + common.UtilsImageEnv: o.UtilsImage, + common.InitImageEnv: o.InitImage, + common.BYOOOTelCollectorImageEnv: o.CollectorImage, + }), + ICMSEnvironment: "perf", + CloudProvider: "perf", + Telemetries: o.telemetries(), + } + return msg +} + +func helmMessage(o Options) function.CreationQueueMessage { + msg := baseMessage() + msg.LaunchSpecification = &function.LaunchSpecification{ + EnvironmentB64: encodeTextEnv(map[string]string{ + common.UtilsImageEnv: o.UtilsImage, + common.InitImageEnv: o.InitImage, + common.BYOOOTelCollectorImageEnv: o.CollectorImage, + }), + ICMSEnvironment: "perf", + CloudProvider: "perf", + Telemetries: o.telemetries(), + HelmChartLaunchSpecification: &common.HelmChartLaunchSpecification{ + HelmChartURL: DefaultHelmChartURL, + }, + } + return msg +} + +// TranslateConfig returns the NVCA-equivalent translate config for the shape. +func TranslateConfig(shape Shape, o Options) function.TranslateConfig { + tc := common.TranslateConfig{ + Namespace: o.Namespace, + ObjectNameBase: o.ObjectNameBase, + InstanceTypeLabelSelectorKey: InstanceTypeLabelKey, + ClusterRegion: o.ClusterRegion, + ClusterName: o.ClusterName, + } + if shape == ShapeContainer { + // Container translation requires a non-zero nvidia.com/* GPU resource. + tc.WorkloadResources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"nvidia.com/gpu": resource.MustParse("1")}, + } + } + return function.TranslateConfig{TranslateConfig: tc} +} + +// encodeTextEnv encodes a KEY=VALUE environment block the way NVCA supplies it +// in LaunchSpecification.EnvironmentB64: sorted lines, base64-encoded. +func encodeTextEnv(env map[string]string) string { + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + + lines := make([]string, 0, len(keys)) + for _, k := range keys { + lines = append(lines, fmt.Sprintf("%s=%s", k, env[k])) + } + return base64.StdEncoding.EncodeToString([]byte(strings.Join(lines, "\n"))) +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec_test.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec_test.go new file mode 100644 index 000000000..5af7c563b --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec_test.go @@ -0,0 +1,105 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" +) + +func TestContainerMessageEnvironment(t *testing.T) { + o := DefaultOptions() + msg, err := Message(ShapeContainer, o) + if err != nil { + t.Fatalf("Message: %v", err) + } + if msg.LaunchSpecification == nil { + t.Fatal("launch specification is nil") + } + if msg.LaunchSpecification.HelmChartLaunchSpecification != nil { + t.Error("container message should not carry a helm chart launch spec") + } + if msg.LaunchSpecification.Telemetries == nil { + t.Error("telemetries should be set so the collector is injected") + } + + env := msg.LaunchSpecification.EnvironmentB64 + for _, key := range []string{ + common.ContainerFunctionImageEnv, + common.UtilsImageEnv, + common.InitImageEnv, + common.BYOOOTelCollectorImageEnv, + } { + v, err := common.GetEncodedVarByKey(env, key) + if err != nil { + t.Fatalf("decode env %s: %v", key, err) + } + if v == "" { + t.Errorf("env %s is empty", key) + } + } + + got, _ := common.GetEncodedVarByKey(env, common.BYOOOTelCollectorImageEnv) + if got != o.CollectorImage { + t.Errorf("collector image env = %q, want %q", got, o.CollectorImage) + } +} + +func TestHelmMessageEnvironment(t *testing.T) { + o := DefaultOptions() + msg, err := Message(ShapeHelm, o) + if err != nil { + t.Fatalf("Message: %v", err) + } + if msg.LaunchSpecification.HelmChartLaunchSpecification == nil { + t.Fatal("helm message must carry a helm chart launch spec") + } + if msg.LaunchSpecification.HelmChartLaunchSpecification.HelmChartURL == "" { + t.Error("helm chart URL is empty") + } + + // Helm functions do not carry an inference container image. + env := msg.LaunchSpecification.EnvironmentB64 + if v, _ := common.GetEncodedVarByKey(env, common.ContainerFunctionImageEnv); v != "" { + t.Errorf("helm message unexpectedly set inference image = %q", v) + } + if v, _ := common.GetEncodedVarByKey(env, common.BYOOOTelCollectorImageEnv); v == "" { + t.Error("helm message missing collector image env") + } +} + +func TestUnknownShapeErrors(t *testing.T) { + if _, err := Message("bogus", DefaultOptions()); err == nil { + t.Error("expected error for unknown shape") + } +} + +func TestTranslateConfigGPUOnlyForContainer(t *testing.T) { + o := DefaultOptions() + + c := TranslateConfig(ShapeContainer, o) + if len(c.WorkloadResources.Limits) == 0 { + t.Error("container translate config must request a GPU resource") + } + + h := TranslateConfig(ShapeHelm, o) + if len(h.WorkloadResources.Limits) != 0 { + t.Error("helm translate config should not set workload GPU resources") + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate.go new file mode 100644 index 000000000..ca41c497a --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate.go @@ -0,0 +1,232 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package validate is the render-shape validation gate. It asserts that the +// translated workload contains the BYOO collector with the expected placement, +// image, resources, ports, volumes, environment, probes, and config wiring. If +// the translator output drifts in a way that would break the test shape, this +// fails clearly so the suite never deploys or measures the wrong thing. +package validate + +import ( + "fmt" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/render" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +// Expectations describes what the collector container must look like. +type Expectations struct { + // Image, when non-empty, is the exact collector image reference expected. + Image string + // Resources, when set, is the expected collector resource requirements. + Resources corev1.ResourceRequirements +} + +// requiredPorts are the collector container ports (name -> containerPort). +var requiredPorts = map[string]int32{ + "otlp-grpc": 14357, + "otlp-http": 14358, + "metrics": 18888, + "health": 13133, + "byoo-metrics": 19090, +} + +// requiredMounts are the collector volume mounts (name -> mountPath). +var requiredMounts = map[string]string{ + "otel-config-data": "/etc/otel", + "otel-collector-secret-data": "/etc/byoo-otel-collector/secrets", + "ess-data": "/config/ess-agent", + "secret-data": "/var/secrets", +} + +const ( + healthProbePath = "/health" + healthProbePort = 13133 +) + +// Render validates placement and the collector container for a rendered result. +// It returns a single aggregated error describing every problem found, or nil. +func Render(r *render.Result, exp Expectations) error { + var errs []string + + switch r.Shape { + case spec.ShapeContainer: + if !r.HasContainer("inference") { + errs = append(errs, `container shape: expected an "inference" container alongside the collector`) + } + if r.OwnerPod == common.UtilsPodName { + errs = append(errs, fmt.Sprintf("container shape: collector unexpectedly placed on %q pod", common.UtilsPodName)) + } + case spec.ShapeHelm: + if r.OwnerPod != common.UtilsPodName { + errs = append(errs, fmt.Sprintf("helm shape: collector expected on %q pod, found on %q", common.UtilsPodName, r.OwnerPod)) + } + if r.Service == nil { + errs = append(errs, "helm shape: expected a byoo-otel-collector OTLP Service") + } + default: + errs = append(errs, fmt.Sprintf("unknown shape %q", r.Shape)) + } + + if err := Collector(r.Collector, r.OTelVersion, exp); err != nil { + errs = append(errs, err.Error()) + } + + if len(errs) > 0 { + return fmt.Errorf("render-shape validation failed for %s shape:\n - %s", r.Shape, strings.Join(errs, "\n - ")) + } + return nil +} + +// Collector validates a single BYOO collector container against expectations. +func Collector(c corev1.Container, otelVersion string, exp Expectations) error { + var errs []string + + if c.Name != common.ByooOTelCollectorPodNameBase { + errs = append(errs, fmt.Sprintf("container name = %q, want %q", c.Name, common.ByooOTelCollectorPodNameBase)) + } + if exp.Image != "" && c.Image != exp.Image { + errs = append(errs, fmt.Sprintf("image = %q, want %q", c.Image, exp.Image)) + } + + gotPorts := map[string]int32{} + for _, p := range c.Ports { + gotPorts[p.Name] = p.ContainerPort + } + for _, name := range sortedKeys(requiredPorts) { + if gotPorts[name] != requiredPorts[name] { + errs = append(errs, fmt.Sprintf("port %q = %d, want %d", name, gotPorts[name], requiredPorts[name])) + } + } + + if c.ReadinessProbe == nil || c.ReadinessProbe.HTTPGet == nil { + errs = append(errs, "missing readiness probe HTTPGet handler") + } else { + if c.ReadinessProbe.HTTPGet.Path != healthProbePath { + errs = append(errs, fmt.Sprintf("readiness probe path = %q, want %q", c.ReadinessProbe.HTTPGet.Path, healthProbePath)) + } + if c.ReadinessProbe.HTTPGet.Port.IntValue() != healthProbePort { + errs = append(errs, fmt.Sprintf("readiness probe port = %d, want %d", c.ReadinessProbe.HTTPGet.Port.IntValue(), healthProbePort)) + } + } + + gotMounts := map[string]string{} + for _, m := range c.VolumeMounts { + gotMounts[m.Name] = m.MountPath + } + for _, name := range sortedKeys(requiredMounts) { + if gotMounts[name] != requiredMounts[name] { + errs = append(errs, fmt.Sprintf("volume mount %q = %q, want %q", name, gotMounts[name], requiredMounts[name])) + } + } + + if exp.Resources.Requests != nil || exp.Resources.Limits != nil { + errs = append(errs, resourceDiffs(c.Resources, exp.Resources)...) + } + + envKeys := map[string]bool{} + for _, e := range c.Env { + envKeys[e.Name] = true + } + if !envKeys["OTEL_EXPORTER_OTLP_ENDPOINT"] { + errs = append(errs, "missing env OTEL_EXPORTER_OTLP_ENDPOINT") + } + + switch otelVersion { + case "v2": + if !argsContain(c.Args, "--telemetries") { + errs = append(errs, "v2 collector args missing --telemetries") + } else if argValue(c.Args, "--telemetries") == "" { + errs = append(errs, "v2 collector --telemetries payload is empty") + } + if !argsContain(c.Args, "--otel-config-path") { + errs = append(errs, "v2 collector args missing --otel-config-path") + } + default: + if !argsContain(c.Args, "--config") { + errs = append(errs, "v1 collector args missing --config") + } + } + + if len(errs) > 0 { + return fmt.Errorf("collector container invalid:\n - %s", strings.Join(errs, "\n - ")) + } + return nil +} + +func resourceDiffs(got, want corev1.ResourceRequirements) []string { + var errs []string + for _, kind := range []struct { + name string + g, w corev1.ResourceList + }{ + {"requests", got.Requests, want.Requests}, + {"limits", got.Limits, want.Limits}, + } { + for _, res := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { + wv, wok := kind.w[res] + if !wok { + continue + } + gv, gok := kind.g[res] + if !gok { + errs = append(errs, fmt.Sprintf("resources.%s.%s missing, want %s", kind.name, res, wv.String())) + continue + } + if gv.Cmp(wv) != 0 { + errs = append(errs, fmt.Sprintf("resources.%s.%s = %s, want %s", kind.name, res, gv.String(), wv.String())) + } + } + } + return errs +} + +func argsContain(args []string, flag string) bool { + for _, a := range args { + if a == flag { + return true + } + } + return false +} + +// argValue returns the token following the given flag in a "--flag value" args +// slice, or "" if not present or has no value. +func argValue(args []string, flag string) string { + for i, a := range args { + if a == flag && i+1 < len(args) { + return args[i+1] + } + } + return "" +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate_test.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate_test.go new file mode 100644 index 000000000..3cd81d213 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate_test.go @@ -0,0 +1,94 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validate + +import ( + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/render" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +func expectations(o spec.Options) Expectations { + return Expectations{ + Image: o.CollectorImage, + Resources: common.GetDefaultContainerResourcesBYOO(), + } +} + +func TestValidateContainerShapePasses(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + if err := Render(res, expectations(o)); err != nil { + t.Fatalf("expected valid container shape, got: %v", err) + } +} + +func TestValidateHelmShapePasses(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeHelm, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + if err := Render(res, expectations(o)); err != nil { + t.Fatalf("expected valid helm shape, got: %v", err) + } +} + +func TestValidateDetectsWrongImage(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + exp := expectations(o) + exp.Image = "registry.invalid/wrong:0.0.1" + if err := Render(res, exp); err == nil { + t.Error("expected validation to detect image mismatch") + } +} + +func TestValidateDetectsMissingPort(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + // Simulate translator drift: drop all but the first collector port. + res.Collector.Ports = res.Collector.Ports[:1] + if err := Collector(res.Collector, res.OTelVersion, expectations(o)); err == nil { + t.Error("expected validation to detect missing collector ports") + } +} + +func TestValidateDetectsMissingReadinessProbe(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + res.Collector.ReadinessProbe = nil + if err := Collector(res.Collector, res.OTelVersion, expectations(o)); err == nil { + t.Error("expected validation to detect missing readiness probe") + } +} From c84eff8e5ecedff4e89813b35d0e3c3af9d3b739 Mon Sep 17 00:00:00 2001 From: shobham Date: Fri, 24 Jul 2026 17:48:00 +0530 Subject: [PATCH 2/2] chore(byoo): bump collector VERSION for perf module addition The perf suite lives under src/compute-plane-services/byoo-otel-collector/, so the version-guard CI check requires a VERSION bump alongside it. Relates to #416. Closes #417. Signed-off-by: shobham --- src/compute-plane-services/byoo-otel-collector/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compute-plane-services/byoo-otel-collector/VERSION b/src/compute-plane-services/byoo-otel-collector/VERSION index a95410b17..4397dd171 100644 --- a/src/compute-plane-services/byoo-otel-collector/VERSION +++ b/src/compute-plane-services/byoo-otel-collector/VERSION @@ -1 +1 @@ -0.157.0 +0.158.0