diff --git a/src/compute-plane-services/byoo-otel-collector/VERSION b/src/compute-plane-services/byoo-otel-collector/VERSION index a95410b17..4397dd171 100644 --- a/src/compute-plane-services/byoo-otel-collector/VERSION +++ b/src/compute-plane-services/byoo-otel-collector/VERSION @@ -1 +1 @@ -0.157.0 +0.158.0 diff --git a/src/compute-plane-services/byoo-otel-collector/perf/README.md b/src/compute-plane-services/byoo-otel-collector/perf/README.md new file mode 100644 index 000000000..bccb31676 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/README.md @@ -0,0 +1,67 @@ +# BYOO Collector Performance Test Suite + +A repeatable performance test suite for the BYOO OpenTelemetry collector. It +exercises the collector under controlled telemetry load using the same workload +shape produced in production, by rendering through the shared translation +library (`icms-translate`) rather than hand-written collector manifests. + +> Status: first milestone. Only rendering and render-shape validation are +> implemented today. Deployment, load generation, the OTLP sink, measurements, +> and reporting land in later milestones. + +## Why translation-driven + +The collector spec and config are generated by calling +`function.Translate(msg, tcfg)` from the shared translation library, so the +suite tests the identical collector spec and config that would be deployed: + +- Container functions get the collector as a workload sidecar. +- Helm functions get the collector on the generated utils pod, fronted by a + ClusterIP OTLP Service. + +The suite extracts the authentic collector container and, for deployment, runs +it with lightweight stand-ins for the non-collector containers, which are not +what the suite measures. + +## Layout + +- `cmd/perf` — CLI entrypoint (`render`, `run`, `cleanup`). +- `pkg/spec` — synthetic container/Helm launch specs and translate config. +- `pkg/render` — runs `function.Translate` and extracts the collector. +- `pkg/validate` — render-shape validation gate. +- `pkg/profile` — `dev` and `baseline` execution profiles. + +This is a standalone Go module. Run its Go commands with `GOWORK=off`: + +```bash +GOWORK=off go build ./... +GOWORK=off go test ./... +``` + +## Usage + +```bash +# Render and validate both shapes (no cluster required). +GOWORK=off go run ./cmd/perf render --shape both + +# Render one shape and print the deployable workload as YAML. +GOWORK=off go run ./cmd/perf render --shape container --output yaml +``` + +### `render` flags + +- `--shape` (`both`): `container`, `helm`, or `both`. +- `--profile` (`dev`): `dev` or `baseline`. +- `--collector-image`: BYOO collector image reference. +- `--namespace` (`byoo-perf`): namespace for rendered objects. +- `--output` (`summary`): `summary`, `yaml`, or `json`. + +`run` and `cleanup` are scaffolded and return a clear "not implemented" message. + +## Execution profiles + +- `dev`: short run (10s warmup, 30s window, 1 repetition) for wiring and local + iteration. +- `baseline`: longer, repeatable run (60s warmup, 5m window, 3 repetitions). + +All profile values are documented defaults and will be overridable per run. diff --git a/src/compute-plane-services/byoo-otel-collector/perf/cmd/perf/main.go b/src/compute-plane-services/byoo-otel-collector/perf/cmd/perf/main.go new file mode 100644 index 000000000..1fba38354 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/cmd/perf/main.go @@ -0,0 +1,207 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command perf is the entrypoint for the BYOO collector performance test +// suite. In this first milestone only "render" is fully wired; "run" and +// "cleanup" are scaffolding for the deployment/load milestones (S4+). +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/render" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + var err error + switch os.Args[1] { + case "render": + err = cmdRender(os.Args[2:]) + case "run": + err = cmdRun(os.Args[2:]) + case "cleanup": + err = cmdCleanup(os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "unknown command %q\n\n", os.Args[1]) + usage() + os.Exit(2) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `perf - BYOO OpenTelemetry collector performance test suite + +Usage: + perf [flags] + +Commands: + render Render the production workload shape via icms-translate and validate it (no cluster required). + run Deploy, drive load, and measure (not yet implemented; see S4+). + cleanup Remove test resources (not yet implemented; see S5+). + +Run "perf -h" for command flags. +`) +} + +func cmdRender(args []string) error { + fs := flag.NewFlagSet("render", flag.ContinueOnError) + shapeFlag := fs.String("shape", "both", `deployment shape: "container", "helm", or "both"`) + profileFlag := fs.String("profile", "dev", `execution profile: "dev" or "baseline"`) + collectorImage := fs.String("collector-image", spec.DefaultCollectorImage, "BYOO collector image reference") + namespace := fs.String("namespace", "byoo-perf", "namespace for rendered objects") + output := fs.String("output", "summary", `output format: "summary", "yaml", or "json"`) + if err := fs.Parse(args); err != nil { + return err + } + + prof, err := profile.Lookup(*profileFlag) + if err != nil { + return err + } + shapes, err := shapesFromFlag(*shapeFlag) + if err != nil { + return err + } + + opts := spec.DefaultOptions() + opts.Namespace = *namespace + opts.CollectorImage = *collectorImage + + exp := validate.Expectations{ + Image: opts.CollectorImage, + Resources: common.GetDefaultContainerResourcesBYOO(), + } + + fmt.Printf("profile=%s warmup=%s window=%s reps=%d\n\n", prof.Name, prof.Warmup, prof.MeasurementWindow, prof.Repetitions) + + for _, shape := range shapes { + res, err := render.Render(shape, opts) + if err != nil { + return fmt.Errorf("render %s: %w", shape, err) + } + if err := validate.Render(res, exp); err != nil { + return err + } + + switch *output { + case "summary": + printSummary(res) + case "yaml", "json": + if err := printObjects(res, *namespace, *output); err != nil { + return err + } + default: + return fmt.Errorf("unknown output %q (want \"summary\", \"yaml\", or \"json\")", *output) + } + } + return nil +} + +func printSummary(res *render.Result) { + fmt.Printf("[%s] VALID\n", res.Shape) + fmt.Printf(" collector image : %s\n", res.Collector.Image) + fmt.Printf(" config version : %s\n", res.OTelVersion) + fmt.Printf(" owner pod : %s\n", res.OwnerPod) + if res.Service != nil { + fmt.Printf(" otlp service : %s\n", res.Service.Name) + } + fmt.Printf(" ports : %s\n", portSummary(res)) + fmt.Printf(" objects : %d translated\n\n", len(res.Objects)) +} + +func portSummary(res *render.Result) string { + parts := make([]string, 0, len(res.Collector.Ports)) + for _, p := range res.Collector.Ports { + parts = append(parts, fmt.Sprintf("%s:%d", p.Name, p.ContainerPort)) + } + return strings.Join(parts, " ") +} + +func printObjects(res *render.Result, namespace, format string) error { + pod := res.BenchPod(namespace) + out, err := yaml.Marshal(pod) + if err != nil { + return fmt.Errorf("marshal bench pod: %w", err) + } + if format == "json" { + out, err = yaml.YAMLToJSON(out) + if err != nil { + return fmt.Errorf("convert to json: %w", err) + } + } + fmt.Printf("# shape=%s benchmark workload (authentic collector + emptyDir stand-ins)\n", res.Shape) + fmt.Printf("%s\n", out) + return nil +} + +func cmdRun(args []string) error { + fs := flag.NewFlagSet("run", flag.ContinueOnError) + _ = fs.String("shape", "both", `deployment shape: "container", "helm", or "both"`) + _ = fs.String("profile", "dev", `execution profile: "dev" or "baseline"`) + _ = fs.String("mode", "k3d", `deployment mode: "k3d" or "remote"`) + _ = fs.Bool("retain", false, "retain test resources for debugging instead of cleaning up") + if err := fs.Parse(args); err != nil { + return err + } + return fmt.Errorf("`run` is not implemented yet; it lands with the deployment/load milestones (S4-S9)") +} + +func cmdCleanup(args []string) error { + fs := flag.NewFlagSet("cleanup", flag.ContinueOnError) + _ = fs.String("mode", "k3d", `deployment mode: "k3d" or "remote"`) + _ = fs.String("namespace", "byoo-perf", "namespace to clean up") + if err := fs.Parse(args); err != nil { + return err + } + return fmt.Errorf("`cleanup` is not implemented yet; it lands with the deployment milestone (S5)") +} + +func shapesFromFlag(s string) ([]spec.Shape, error) { + switch s { + case "container": + return []spec.Shape{spec.ShapeContainer}, nil + case "helm": + return []spec.Shape{spec.ShapeHelm}, nil + case "both": + return []spec.Shape{spec.ShapeContainer, spec.ShapeHelm}, nil + default: + return nil, fmt.Errorf("unknown shape %q (want \"container\", \"helm\", or \"both\")", s) + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/go.mod b/src/compute-plane-services/byoo-otel-collector/perf/go.mod new file mode 100644 index 000000000..7726b59f0 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/go.mod @@ -0,0 +1,50 @@ +module github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf + +go 1.25.0 + +require ( + github.com/NVIDIA/nvcf/src/libraries/go/lib v0.0.0 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/go-containerregistry v0.21.5 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/text v0.37.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect +) + +// The perf suite consumes the shared SIS translation library directly so that +// it renders exactly the workload shape NVCA produces in production. +replace github.com/NVIDIA/nvcf/src/libraries/go/lib => ../../../libraries/go/lib + +// Mirror the k8s pins used by the shared library. replace directives in +// dependencies are not inherited, so we repeat them here for the main module. +replace ( + k8s.io/api => k8s.io/api v0.34.2 + k8s.io/apimachinery => k8s.io/apimachinery v0.34.2 + k8s.io/client-go => k8s.io/client-go v0.34.2 +) diff --git a/src/compute-plane-services/byoo-otel-collector/perf/go.sum b/src/compute-plane-services/byoo-otel-collector/perf/go.sum new file mode 100644 index 000000000..e60a5b515 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/go.sum @@ -0,0 +1,113 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.5 h1:KTJG9Pn/jC0VdZR6ctV3/jcN+q6/Iqlx0sTVz3ywZlM= +github.com/google/go-containerregistry v0.21.5/go.mod h1:ySvMuiWg+dOsRW0Hw8GYwfMwBlNRTmpYBFJPlkco5zU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= +k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= +k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= +k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile/profile.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile/profile.go new file mode 100644 index 000000000..9ec659028 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/profile/profile.go @@ -0,0 +1,80 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package profile defines the named execution profiles for the BYOO collector +// performance suite. Profiles bundle the run-shape knobs (warmup, measurement +// window, repetitions, and default load rates). All values are overridable via +// CLI flags; the constants here are documented defaults only. +package profile + +import ( + "fmt" + "time" +) + +// Profile is a named bundle of run parameters. +type Profile struct { + // Name is the profile identifier ("dev" or "baseline"). + Name string + // Warmup is the duration load runs before the measurement window opens. + Warmup time.Duration + // MeasurementWindow is the duration over which measurements are recorded. + MeasurementWindow time.Duration + // Repetitions is how many times the measurement window is repeated. + Repetitions int + // LogRecordsPerSec is the default logs load rate. + LogRecordsPerSec int + // MetricDataPointsPerSec is the default metrics load rate. + MetricDataPointsPerSec int +} + +// Dev is a short run intended to validate wiring and support local iteration. +func Dev() Profile { + return Profile{ + Name: "dev", + Warmup: 10 * time.Second, + MeasurementWindow: 30 * time.Second, + Repetitions: 1, + LogRecordsPerSec: 1000, + MetricDataPointsPerSec: 1000, + } +} + +// Baseline is a longer, repeatable run with warmup, a defined measurement +// window, and repeated measurements. +func Baseline() Profile { + return Profile{ + Name: "baseline", + Warmup: 60 * time.Second, + MeasurementWindow: 5 * time.Minute, + Repetitions: 3, + LogRecordsPerSec: 10000, + MetricDataPointsPerSec: 10000, + } +} + +// Lookup returns the named profile, or an error if the name is unknown. +func Lookup(name string) (Profile, error) { + switch name { + case "dev": + return Dev(), nil + case "baseline": + return Baseline(), nil + default: + return Profile{}, fmt.Errorf("unknown profile %q (want \"dev\" or \"baseline\")", name) + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render.go new file mode 100644 index 000000000..ccd5568cf --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render.go @@ -0,0 +1,156 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package render drives the shared SIS translation library to produce the +// production workload shape, then extracts the authentic BYOO collector for +// the performance suite. Rendering touches no cluster; deployment happens +// later (S5/S6). +package render + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +// CollectorContainerName is the container name the translator assigns the BYOO +// collector in every workload shape. +const CollectorContainerName = common.ByooOTelCollectorPodNameBase + +// Result is the outcome of rendering a workload shape through the translator. +type Result struct { + Shape spec.Shape + Options spec.Options + + // Objects is the full set of translated Kubernetes objects. + Objects []metav1.Object + // Collector is the authentic BYOO collector container, exactly as the + // translator emitted it. + Collector corev1.Container + // OwnerPod is the name of the pod that hosts the collector. + OwnerPod string + // Service is the OTLP ClusterIP Service (Helm shape only; nil otherwise). + Service *corev1.Service + // OTelVersion is the collector config code path ("v1" or "v2") the + // translator selected from the collector image tag. + OTelVersion string +} + +// Render builds the synthetic spec for the shape, runs it through +// function.Translate, and extracts the authentic collector container. +func Render(shape spec.Shape, o spec.Options) (*Result, error) { + msg, err := spec.Message(shape, o) + if err != nil { + return nil, err + } + tcfg := spec.TranslateConfig(shape, o) + + objs, err := function.Translate(msg, tcfg) + if err != nil { + return nil, fmt.Errorf("translate %s function: %w", shape, err) + } + + res := &Result{ + Shape: shape, + Options: o, + Objects: objs, + OTelVersion: common.OTelVersion(o.CollectorImage), + } + + found := false + for _, obj := range objs { + switch t := obj.(type) { + case *corev1.Pod: + for i := range t.Spec.Containers { + if t.Spec.Containers[i].Name == CollectorContainerName { + res.Collector = *t.Spec.Containers[i].DeepCopy() + res.OwnerPod = t.Name + found = true + } + } + case *corev1.Service: + if t.Name == common.ByooOTelCollectorPodNameBase { + res.Service = t.DeepCopy() + } + } + } + if !found { + return nil, fmt.Errorf("no %q container found in translated %s workload (are telemetries enabled?)", CollectorContainerName, shape) + } + return res, nil +} + +// HasContainer reports whether any translated pod contains a container with the +// given name (init containers included). +func (r *Result) HasContainer(name string) bool { + for _, obj := range r.Objects { + pod, ok := obj.(*corev1.Pod) + if !ok { + continue + } + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == name { + return true + } + } + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == name { + return true + } + } + } + return false +} + +// BenchPod produces the deployable performance workload: the authentic +// collector container plus lightweight emptyDir stand-ins for every volume it +// mounts. The non-collector function containers (init, GPU inference, utils, +// ESS) are intentionally omitted since they cannot run on k3d and are not what +// the suite measures. Full deployment orchestration is added in S5/S6. +func (r *Result) BenchPod(namespace string) *corev1.Pod { + pod := &corev1.Pod{} + pod.Name = r.Options.ObjectNameBase + "-collector" + pod.Namespace = namespace + pod.Labels = map[string]string{ + common.K8sAppNameLabelKey: common.ByooOTelCollectorPodNameBase, + "app.kubernetes.io/part-of": "byoo-perf", + common.BYOOMetricsEgressTargetLabelKey: common.BYOOMetricsEgressTargetLabelValue, + } + + collector := *r.Collector.DeepCopy() + pod.Spec.Containers = []corev1.Container{collector} + pod.Spec.RestartPolicy = corev1.RestartPolicyNever + + seen := map[string]bool{} + for _, vm := range collector.VolumeMounts { + if seen[vm.Name] { + continue + } + seen[vm.Name] = true + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: vm.Name, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) + } + return pod +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render_test.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render_test.go new file mode 100644 index 000000000..b94a74bbc --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/render/render_test.go @@ -0,0 +1,88 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package render + +import ( + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +func TestRenderContainerExtractsSidecar(t *testing.T) { + res, err := Render(spec.ShapeContainer, spec.DefaultOptions()) + if err != nil { + t.Fatalf("Render: %v", err) + } + if res.Collector.Name != common.ByooOTelCollectorPodNameBase { + t.Errorf("collector name = %q, want %q", res.Collector.Name, common.ByooOTelCollectorPodNameBase) + } + if res.Collector.Image != spec.DefaultCollectorImage { + t.Errorf("collector image = %q, want %q", res.Collector.Image, spec.DefaultCollectorImage) + } + if res.OTelVersion != "v2" { + t.Errorf("otel version = %q, want v2 for the default (>0.119.4) image", res.OTelVersion) + } + if res.OwnerPod != "0-perf" { + t.Errorf("owner pod = %q, want %q", res.OwnerPod, "0-perf") + } + if !res.HasContainer("inference") { + t.Error("container shape must include an inference container") + } + if res.Service != nil { + t.Error("container shape should not produce an OTLP Service") + } +} + +func TestRenderHelmPlacesOnUtilsPodWithService(t *testing.T) { + res, err := Render(spec.ShapeHelm, spec.DefaultOptions()) + if err != nil { + t.Fatalf("Render: %v", err) + } + if res.OwnerPod != common.UtilsPodName { + t.Errorf("owner pod = %q, want %q", res.OwnerPod, common.UtilsPodName) + } + if res.Service == nil { + t.Fatal("helm shape must produce an OTLP Service") + } + if res.Service.Name != common.ByooOTelCollectorPodNameBase { + t.Errorf("service name = %q, want %q", res.Service.Name, common.ByooOTelCollectorPodNameBase) + } +} + +func TestBenchPodSuppliesVolumesForEveryMount(t *testing.T) { + res, err := Render(spec.ShapeContainer, spec.DefaultOptions()) + if err != nil { + t.Fatalf("Render: %v", err) + } + pod := res.BenchPod("byoo-perf") + if len(pod.Spec.Containers) != 1 { + t.Fatalf("bench pod containers = %d, want 1 (authentic collector only)", len(pod.Spec.Containers)) + } + + vols := map[string]bool{} + for _, v := range pod.Spec.Volumes { + vols[v.Name] = true + } + for _, m := range pod.Spec.Containers[0].VolumeMounts { + if !vols[m.Name] { + t.Errorf("collector mount %q has no backing volume in the bench pod", m.Name) + } + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec.go new file mode 100644 index 000000000..ec5827c9e --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec.go @@ -0,0 +1,228 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package spec builds synthetic NVCF function launch specifications and the +// NVCA-equivalent translate configuration used to drive the shared SIS +// translation library. Feeding these through function.Translate yields the +// exact workload shape NVCA produces in production, including BYOO collector +// placement. +package spec + +import ( + "encoding/base64" + "fmt" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" +) + +// Shape is a supported NVCF function deployment shape. +type Shape string + +const ( + // ShapeContainer is a container function; the BYOO collector is a workload + // sidecar on the inference pod. + ShapeContainer Shape = "container" + // ShapeHelm is a Helm function; the BYOO collector runs on the generated + // utils pod and is fronted by a ClusterIP Service. + ShapeHelm Shape = "helm" +) + +// Default image and identifier values for synthetic specs. The collector image +// tag must be greater than the icms-translate v2 cutoff (0.119.4) so the +// rendered collector uses the runtime-config ("v2") code path that matches +// current production. +const ( + DefaultCollectorImage = "nvcr.io/nvidia/nvcf-core/byoo-otel-collector:0.153.1" + DefaultInferenceImage = "nvcr.io/nvidia/nvcf-perf/stub-inference:latest" + DefaultUtilsImage = "nvcr.io/nvidia/nvcf-core/worker-utils:latest" + DefaultInitImage = "nvcr.io/nvidia/nvcf-core/worker-init:latest" + DefaultHelmChartURL = "https://helm.example.invalid/charts/perf-test-chart-0.1.0.tgz" + + // InstanceTypeLabelKey mirrors the label selector key NVCA uses for + // instance-type node affinity. + InstanceTypeLabelKey = "node.kubernetes.io/instance-type" + + instanceTypeValue = "perf.gpu" +) + +// Options controls the synthetic launch specification and translate config. +type Options struct { + Namespace string + ObjectNameBase string + + CollectorImage string + InferenceImage string + UtilsImage string + InitImage string + + ClusterRegion string + ClusterName string + + // Telemetry destination knobs. These are placeholder endpoints for + // rendering; deployment (S5/S6) overrides them to point at the in-cluster + // OTLP sink. + Protocol string + Provider string + LogsEndpoint string + MetricsEndpoint string + EnableLogs bool + EnableMetrics bool +} + +// DefaultOptions returns a self-consistent set of options for local rendering. +func DefaultOptions() Options { + return Options{ + Namespace: "byoo-perf", + ObjectNameBase: "perf", + CollectorImage: DefaultCollectorImage, + InferenceImage: DefaultInferenceImage, + UtilsImage: DefaultUtilsImage, + InitImage: DefaultInitImage, + ClusterRegion: "perf-local", + ClusterName: "perf", + Protocol: "grpc", + Provider: "GRAFANA_CLOUD", + LogsEndpoint: "https://logs.perf.invalid/otlp", + MetricsEndpoint: "https://metrics.perf.invalid/otlp", + EnableLogs: true, + EnableMetrics: true, + } +} + +func (o Options) telemetries() *common.TelemetriesLaunchSpecification { + if !o.EnableLogs && !o.EnableMetrics { + return nil + } + t := &common.TelemetriesLaunchSpecification{} + if o.EnableLogs { + t.Telemetries.Logs = &common.Telemetry{ + Protocol: o.Protocol, Provider: o.Provider, Endpoint: o.LogsEndpoint, Name: "perf-logs", + } + } + if o.EnableMetrics { + t.Telemetries.Metrics = &common.Telemetry{ + Protocol: o.Protocol, Provider: o.Provider, Endpoint: o.MetricsEndpoint, Name: "perf-metrics", + } + } + return t +} + +// Message builds the synthetic creation-queue message for the given shape. +func Message(shape Shape, o Options) (function.CreationQueueMessage, error) { + switch shape { + case ShapeContainer: + return containerMessage(o), nil + case ShapeHelm: + return helmMessage(o), nil + default: + return function.CreationQueueMessage{}, fmt.Errorf("unknown shape %q (want %q or %q)", shape, ShapeContainer, ShapeHelm) + } +} + +func baseMessage() function.CreationQueueMessage { + return function.CreationQueueMessage{ + CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{ + Action: common.FunctionCreationAction, + RequestID: "perf-request", + MessageBatchID: "perf-batch", + NCAID: "perf-nca", + InstanceCount: 1, + InstanceTypeValue: instanceTypeValue, + GPUType: "PERF", + RequestedGPUCount: 1, + }, + Details: function.Details{ + FunctionID: "perf-function", + FunctionVersionID: "perf-version", + FunctionType: function.FunctionTypeDefault, + }, + } +} + +func containerMessage(o Options) function.CreationQueueMessage { + msg := baseMessage() + msg.LaunchSpecification = &function.LaunchSpecification{ + EnvironmentB64: encodeTextEnv(map[string]string{ + common.ContainerFunctionImageEnv: o.InferenceImage, + common.UtilsImageEnv: o.UtilsImage, + common.InitImageEnv: o.InitImage, + common.BYOOOTelCollectorImageEnv: o.CollectorImage, + }), + ICMSEnvironment: "perf", + CloudProvider: "perf", + Telemetries: o.telemetries(), + } + return msg +} + +func helmMessage(o Options) function.CreationQueueMessage { + msg := baseMessage() + msg.LaunchSpecification = &function.LaunchSpecification{ + EnvironmentB64: encodeTextEnv(map[string]string{ + common.UtilsImageEnv: o.UtilsImage, + common.InitImageEnv: o.InitImage, + common.BYOOOTelCollectorImageEnv: o.CollectorImage, + }), + ICMSEnvironment: "perf", + CloudProvider: "perf", + Telemetries: o.telemetries(), + HelmChartLaunchSpecification: &common.HelmChartLaunchSpecification{ + HelmChartURL: DefaultHelmChartURL, + }, + } + return msg +} + +// TranslateConfig returns the NVCA-equivalent translate config for the shape. +func TranslateConfig(shape Shape, o Options) function.TranslateConfig { + tc := common.TranslateConfig{ + Namespace: o.Namespace, + ObjectNameBase: o.ObjectNameBase, + InstanceTypeLabelSelectorKey: InstanceTypeLabelKey, + ClusterRegion: o.ClusterRegion, + ClusterName: o.ClusterName, + } + if shape == ShapeContainer { + // Container translation requires a non-zero nvidia.com/* GPU resource. + tc.WorkloadResources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"nvidia.com/gpu": resource.MustParse("1")}, + } + } + return function.TranslateConfig{TranslateConfig: tc} +} + +// encodeTextEnv encodes a KEY=VALUE environment block the way NVCA supplies it +// in LaunchSpecification.EnvironmentB64: sorted lines, base64-encoded. +func encodeTextEnv(env map[string]string) string { + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + + lines := make([]string, 0, len(keys)) + for _, k := range keys { + lines = append(lines, fmt.Sprintf("%s=%s", k, env[k])) + } + return base64.StdEncoding.EncodeToString([]byte(strings.Join(lines, "\n"))) +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec_test.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec_test.go new file mode 100644 index 000000000..5af7c563b --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec/spec_test.go @@ -0,0 +1,105 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" +) + +func TestContainerMessageEnvironment(t *testing.T) { + o := DefaultOptions() + msg, err := Message(ShapeContainer, o) + if err != nil { + t.Fatalf("Message: %v", err) + } + if msg.LaunchSpecification == nil { + t.Fatal("launch specification is nil") + } + if msg.LaunchSpecification.HelmChartLaunchSpecification != nil { + t.Error("container message should not carry a helm chart launch spec") + } + if msg.LaunchSpecification.Telemetries == nil { + t.Error("telemetries should be set so the collector is injected") + } + + env := msg.LaunchSpecification.EnvironmentB64 + for _, key := range []string{ + common.ContainerFunctionImageEnv, + common.UtilsImageEnv, + common.InitImageEnv, + common.BYOOOTelCollectorImageEnv, + } { + v, err := common.GetEncodedVarByKey(env, key) + if err != nil { + t.Fatalf("decode env %s: %v", key, err) + } + if v == "" { + t.Errorf("env %s is empty", key) + } + } + + got, _ := common.GetEncodedVarByKey(env, common.BYOOOTelCollectorImageEnv) + if got != o.CollectorImage { + t.Errorf("collector image env = %q, want %q", got, o.CollectorImage) + } +} + +func TestHelmMessageEnvironment(t *testing.T) { + o := DefaultOptions() + msg, err := Message(ShapeHelm, o) + if err != nil { + t.Fatalf("Message: %v", err) + } + if msg.LaunchSpecification.HelmChartLaunchSpecification == nil { + t.Fatal("helm message must carry a helm chart launch spec") + } + if msg.LaunchSpecification.HelmChartLaunchSpecification.HelmChartURL == "" { + t.Error("helm chart URL is empty") + } + + // Helm functions do not carry an inference container image. + env := msg.LaunchSpecification.EnvironmentB64 + if v, _ := common.GetEncodedVarByKey(env, common.ContainerFunctionImageEnv); v != "" { + t.Errorf("helm message unexpectedly set inference image = %q", v) + } + if v, _ := common.GetEncodedVarByKey(env, common.BYOOOTelCollectorImageEnv); v == "" { + t.Error("helm message missing collector image env") + } +} + +func TestUnknownShapeErrors(t *testing.T) { + if _, err := Message("bogus", DefaultOptions()); err == nil { + t.Error("expected error for unknown shape") + } +} + +func TestTranslateConfigGPUOnlyForContainer(t *testing.T) { + o := DefaultOptions() + + c := TranslateConfig(ShapeContainer, o) + if len(c.WorkloadResources.Limits) == 0 { + t.Error("container translate config must request a GPU resource") + } + + h := TranslateConfig(ShapeHelm, o) + if len(h.WorkloadResources.Limits) != 0 { + t.Error("helm translate config should not set workload GPU resources") + } +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate.go new file mode 100644 index 000000000..ca41c497a --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate.go @@ -0,0 +1,232 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package validate is the render-shape validation gate. It asserts that the +// translated workload contains the BYOO collector with the expected placement, +// image, resources, ports, volumes, environment, probes, and config wiring. If +// the translator output drifts in a way that would break the test shape, this +// fails clearly so the suite never deploys or measures the wrong thing. +package validate + +import ( + "fmt" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/render" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +// Expectations describes what the collector container must look like. +type Expectations struct { + // Image, when non-empty, is the exact collector image reference expected. + Image string + // Resources, when set, is the expected collector resource requirements. + Resources corev1.ResourceRequirements +} + +// requiredPorts are the collector container ports (name -> containerPort). +var requiredPorts = map[string]int32{ + "otlp-grpc": 14357, + "otlp-http": 14358, + "metrics": 18888, + "health": 13133, + "byoo-metrics": 19090, +} + +// requiredMounts are the collector volume mounts (name -> mountPath). +var requiredMounts = map[string]string{ + "otel-config-data": "/etc/otel", + "otel-collector-secret-data": "/etc/byoo-otel-collector/secrets", + "ess-data": "/config/ess-agent", + "secret-data": "/var/secrets", +} + +const ( + healthProbePath = "/health" + healthProbePort = 13133 +) + +// Render validates placement and the collector container for a rendered result. +// It returns a single aggregated error describing every problem found, or nil. +func Render(r *render.Result, exp Expectations) error { + var errs []string + + switch r.Shape { + case spec.ShapeContainer: + if !r.HasContainer("inference") { + errs = append(errs, `container shape: expected an "inference" container alongside the collector`) + } + if r.OwnerPod == common.UtilsPodName { + errs = append(errs, fmt.Sprintf("container shape: collector unexpectedly placed on %q pod", common.UtilsPodName)) + } + case spec.ShapeHelm: + if r.OwnerPod != common.UtilsPodName { + errs = append(errs, fmt.Sprintf("helm shape: collector expected on %q pod, found on %q", common.UtilsPodName, r.OwnerPod)) + } + if r.Service == nil { + errs = append(errs, "helm shape: expected a byoo-otel-collector OTLP Service") + } + default: + errs = append(errs, fmt.Sprintf("unknown shape %q", r.Shape)) + } + + if err := Collector(r.Collector, r.OTelVersion, exp); err != nil { + errs = append(errs, err.Error()) + } + + if len(errs) > 0 { + return fmt.Errorf("render-shape validation failed for %s shape:\n - %s", r.Shape, strings.Join(errs, "\n - ")) + } + return nil +} + +// Collector validates a single BYOO collector container against expectations. +func Collector(c corev1.Container, otelVersion string, exp Expectations) error { + var errs []string + + if c.Name != common.ByooOTelCollectorPodNameBase { + errs = append(errs, fmt.Sprintf("container name = %q, want %q", c.Name, common.ByooOTelCollectorPodNameBase)) + } + if exp.Image != "" && c.Image != exp.Image { + errs = append(errs, fmt.Sprintf("image = %q, want %q", c.Image, exp.Image)) + } + + gotPorts := map[string]int32{} + for _, p := range c.Ports { + gotPorts[p.Name] = p.ContainerPort + } + for _, name := range sortedKeys(requiredPorts) { + if gotPorts[name] != requiredPorts[name] { + errs = append(errs, fmt.Sprintf("port %q = %d, want %d", name, gotPorts[name], requiredPorts[name])) + } + } + + if c.ReadinessProbe == nil || c.ReadinessProbe.HTTPGet == nil { + errs = append(errs, "missing readiness probe HTTPGet handler") + } else { + if c.ReadinessProbe.HTTPGet.Path != healthProbePath { + errs = append(errs, fmt.Sprintf("readiness probe path = %q, want %q", c.ReadinessProbe.HTTPGet.Path, healthProbePath)) + } + if c.ReadinessProbe.HTTPGet.Port.IntValue() != healthProbePort { + errs = append(errs, fmt.Sprintf("readiness probe port = %d, want %d", c.ReadinessProbe.HTTPGet.Port.IntValue(), healthProbePort)) + } + } + + gotMounts := map[string]string{} + for _, m := range c.VolumeMounts { + gotMounts[m.Name] = m.MountPath + } + for _, name := range sortedKeys(requiredMounts) { + if gotMounts[name] != requiredMounts[name] { + errs = append(errs, fmt.Sprintf("volume mount %q = %q, want %q", name, gotMounts[name], requiredMounts[name])) + } + } + + if exp.Resources.Requests != nil || exp.Resources.Limits != nil { + errs = append(errs, resourceDiffs(c.Resources, exp.Resources)...) + } + + envKeys := map[string]bool{} + for _, e := range c.Env { + envKeys[e.Name] = true + } + if !envKeys["OTEL_EXPORTER_OTLP_ENDPOINT"] { + errs = append(errs, "missing env OTEL_EXPORTER_OTLP_ENDPOINT") + } + + switch otelVersion { + case "v2": + if !argsContain(c.Args, "--telemetries") { + errs = append(errs, "v2 collector args missing --telemetries") + } else if argValue(c.Args, "--telemetries") == "" { + errs = append(errs, "v2 collector --telemetries payload is empty") + } + if !argsContain(c.Args, "--otel-config-path") { + errs = append(errs, "v2 collector args missing --otel-config-path") + } + default: + if !argsContain(c.Args, "--config") { + errs = append(errs, "v1 collector args missing --config") + } + } + + if len(errs) > 0 { + return fmt.Errorf("collector container invalid:\n - %s", strings.Join(errs, "\n - ")) + } + return nil +} + +func resourceDiffs(got, want corev1.ResourceRequirements) []string { + var errs []string + for _, kind := range []struct { + name string + g, w corev1.ResourceList + }{ + {"requests", got.Requests, want.Requests}, + {"limits", got.Limits, want.Limits}, + } { + for _, res := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { + wv, wok := kind.w[res] + if !wok { + continue + } + gv, gok := kind.g[res] + if !gok { + errs = append(errs, fmt.Sprintf("resources.%s.%s missing, want %s", kind.name, res, wv.String())) + continue + } + if gv.Cmp(wv) != 0 { + errs = append(errs, fmt.Sprintf("resources.%s.%s = %s, want %s", kind.name, res, gv.String(), wv.String())) + } + } + } + return errs +} + +func argsContain(args []string, flag string) bool { + for _, a := range args { + if a == flag { + return true + } + } + return false +} + +// argValue returns the token following the given flag in a "--flag value" args +// slice, or "" if not present or has no value. +func argValue(args []string, flag string) string { + for i, a := range args { + if a == flag && i+1 < len(args) { + return args[i+1] + } + } + return "" +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate_test.go b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate_test.go new file mode 100644 index 000000000..3cd81d213 --- /dev/null +++ b/src/compute-plane-services/byoo-otel-collector/perf/pkg/validate/validate_test.go @@ -0,0 +1,94 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validate + +import ( + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/render" + "github.com/NVIDIA/nvcf/src/compute-plane-services/byoo-otel-collector/perf/pkg/spec" +) + +func expectations(o spec.Options) Expectations { + return Expectations{ + Image: o.CollectorImage, + Resources: common.GetDefaultContainerResourcesBYOO(), + } +} + +func TestValidateContainerShapePasses(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + if err := Render(res, expectations(o)); err != nil { + t.Fatalf("expected valid container shape, got: %v", err) + } +} + +func TestValidateHelmShapePasses(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeHelm, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + if err := Render(res, expectations(o)); err != nil { + t.Fatalf("expected valid helm shape, got: %v", err) + } +} + +func TestValidateDetectsWrongImage(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + exp := expectations(o) + exp.Image = "registry.invalid/wrong:0.0.1" + if err := Render(res, exp); err == nil { + t.Error("expected validation to detect image mismatch") + } +} + +func TestValidateDetectsMissingPort(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + // Simulate translator drift: drop all but the first collector port. + res.Collector.Ports = res.Collector.Ports[:1] + if err := Collector(res.Collector, res.OTelVersion, expectations(o)); err == nil { + t.Error("expected validation to detect missing collector ports") + } +} + +func TestValidateDetectsMissingReadinessProbe(t *testing.T) { + o := spec.DefaultOptions() + res, err := render.Render(spec.ShapeContainer, o) + if err != nil { + t.Fatalf("Render: %v", err) + } + res.Collector.ReadinessProbe = nil + if err := Collector(res.Collector, res.OTelVersion, expectations(o)); err == nil { + t.Error("expected validation to detect missing readiness probe") + } +}