From b2c6d6dedc2a2737370a8b0dda150d74ab22d99f Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 16:15:48 +0800 Subject: [PATCH 1/3] feat(sweep): spec provenance stamp, intent compilation, declared ladder trims sweep plan now covers the machine-specific intent that forced hand-authoring (vllm runtime path, GPU memory utilization, KV cache bytes) and accepts declared ladder trims with reasons. Generated specs carry a generator stamp with a canonical content hash; bench run prints the verified provenance and reports label runs Generated default sweep or Custom grid from the stored spec bytes, never on trust. Trimmed points render like adaptive skips with their reason, never as holes. --- README.md | 10 ++ docs/2026-07-02-default-inference-sweep.md | 29 +++++ docs/2026-07-05-report-integrity-plan.md | 4 +- internal/artifact/provenance.go | 72 +++++++++++ internal/benchcli/benchcli.go | 70 +++++++++-- internal/report/html.go | 105 +++++++++++++++- internal/report/html_test.go | 81 +++++++++++++ internal/sweepplan/provenance_test.go | 114 ++++++++++++++++++ internal/sweepplan/sweepplan.go | 113 ++++++++++++++--- .../testdata/default-sweep-8k.golden.json | 21 +++- internal/vllmbench/config.go | 97 +++++++++++++-- internal/vllmbench/config_test.go | 16 +++ 12 files changed, 691 insertions(+), 41 deletions(-) create mode 100644 internal/artifact/provenance.go create mode 100644 internal/sweepplan/provenance_test.go diff --git a/README.md b/README.md index 4873363..391c928 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,16 @@ collides with different provenance is refused instead of silently replaced. The report lists every run and aggregates repeated points across runs with mean ± spread. +## Spec Provenance + +Create specs with `sweep plan`; machine-specific runtime choices are flags +(`--vllm-command`, `--gpu-memory-utilization`, `--kv-cache-memory-bytes`), +and deliberate ladder caps are declared trims with reasons +(`--trim 64k=8:'12 GiB KV budget'`) that render in reports like adaptive +skips. Generated specs carry a verified `generator` stamp: reports label +runs "Generated default sweep" only while the spec's content hash still +matches, and anything hand-written or edited shows as "Custom grid". + ## Context Semantics Every workload must declare what its context number means: diff --git a/docs/2026-07-02-default-inference-sweep.md b/docs/2026-07-02-default-inference-sweep.md index f23de7c..41598b8 100644 --- a/docs/2026-07-02-default-inference-sweep.md +++ b/docs/2026-07-02-default-inference-sweep.md @@ -181,3 +181,32 @@ whose results already completed. Do not call a sweep complete until all completed, skipped, and failed cases are recorded with enough metadata to reproduce the run. + +## Spec Provenance + +Create sweep specs only through the generator; never hand-write or +post-edit the JSON: + +```sh +localperf sweep plan \ + --model \ + --vllm-command /path/to/runtime/bin/vllm \ + --gpu-memory-utilization 0.4 \ + --kv-cache-memory-bytes 12884901888 \ + --trim 64k=8:'12 GiB KV budget' \ + --out spec.json +``` + +Machine-specific runtime choices (vllm path, GPU memory, KV budget, extra +serve args) are generator flags, so nothing forces editing the output. A +deliberate concurrency cap is a declared trim: `--trim =:` +removes the higher points from the grid, records the decision in the spec, +and reports render the trimmed points like adaptive skips — with the reason, +never as silent holes. + +Generated specs carry a `generator` stamp (tool, intent, content hash). +`bench run` verifies the hash and prints the provenance; reports and the +viewer label the run "Generated default sweep" only when the hash still +matches. A spec without a stamp, or edited after generation, runs fine but +is labeled "Custom grid" — the label is verified from the stored spec bytes, +never taken on trust. diff --git a/docs/2026-07-05-report-integrity-plan.md b/docs/2026-07-05-report-integrity-plan.md index 2c30cf8..7898ffd 100644 --- a/docs/2026-07-05-report-integrity-plan.md +++ b/docs/2026-07-05-report-integrity-plan.md @@ -6,8 +6,8 @@ date: 2026-07-05 # Report Integrity Plan -Date: 2026-07-05. Status: PR 1 (streaming TTFT) and PR 2 (rendering + -legacy cutover) implemented; PR 3 (provenance) in progress. +Date: 2026-07-05. Status: implemented (PR 1 streaming TTFT, PR 2 +rendering + legacy cutover, PR 3 spec provenance). Fixes every problem found while inspecting the live viewer serving the 4-model GB10 sweep of 2026-07-05, each verified against the code. diff --git a/internal/artifact/provenance.go b/internal/artifact/provenance.go new file mode 100644 index 0000000..b050920 --- /dev/null +++ b/internal/artifact/provenance.go @@ -0,0 +1,72 @@ +package artifact + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +// GeneratorStamp is the provenance record `sweep plan` writes into generated +// specs. ContentHash covers the canonical spec document with the "generator" +// key removed, so any post-generation edit is detectable; see +// docs/2026-07-05-report-integrity-plan.md. +type GeneratorStamp struct { + Tool string `json:"tool"` + Version string `json:"version"` + Intent json.RawMessage `json:"intent,omitempty"` + LadderTrims []LadderTrim `json:"ladder_trims,omitempty"` + ContentHash string `json:"content_hash"` +} + +// LadderTrim declares an author decision to cap a context's concurrency +// ladder, with the reason rendered in reports so trimmed points never look +// like silent holes. +type LadderTrim struct { + Context int `json:"context"` + MaxConcurrency int `json:"max_concurrency"` + Reason string `json:"reason"` +} + +// Spec provenance statuses; reports label anything but a verified generated +// spec as a custom grid. +const ( + SpecProvenanceGenerated = "generated" + SpecProvenanceEdited = "edited" + SpecProvenanceCustom = "custom" +) + +// CanonicalSpecHash hashes a spec document with the "generator" key removed, +// after canonicalizing through a generic JSON map (sorted keys), so the +// generator and every verifier agree byte-for-byte regardless of struct +// field order. +func CanonicalSpecHash(raw []byte) (string, error) { + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + return "", err + } + delete(doc, "generator") + canonical, err := json.Marshal(doc) + if err != nil { + return "", err + } + sum := sha256.Sum256(canonical) + return hex.EncodeToString(sum[:]), nil +} + +// VerifySpecProvenance checks raw spec bytes against their embedded +// generator stamp: "generated" when the content hash matches, "edited" when +// a stamp exists but the content changed, "custom" when there is no stamp +// or the document does not parse. +func VerifySpecProvenance(raw []byte) (string, *GeneratorStamp) { + var doc struct { + Generator *GeneratorStamp `json:"generator"` + } + if err := json.Unmarshal(raw, &doc); err != nil || doc.Generator == nil { + return SpecProvenanceCustom, nil + } + hash, err := CanonicalSpecHash(raw) + if err != nil || hash != doc.Generator.ContentHash { + return SpecProvenanceEdited, doc.Generator + } + return SpecProvenanceGenerated, doc.Generator +} diff --git a/internal/benchcli/benchcli.go b/internal/benchcli/benchcli.go index 24f6d6c..cb72656 100644 --- a/internal/benchcli/benchcli.go +++ b/internal/benchcli/benchcli.go @@ -86,6 +86,11 @@ func runSweepPlan(args []string) { reference := flags.Bool("reference", true, "include the 4k max-throughput-reference capacity family") stress := flags.Bool("stress", false, "add long-output decode spot checks (4096 tokens at 32k c4, 64k c1/c4) and the 128k points") memFloor := flags.Float64("min-mem-available-gib", 0, "safety memory floor in GiB (default 40)") + vllmCommand := flags.String("vllm-command", "", "vllm executable for managed serves (machine-specific runtime path)") + gpuMemUtil := flags.Float64("gpu-memory-utilization", 0, "gpu memory utilization applied to every profile") + kvCacheBytes := flags.Int64("kv-cache-memory-bytes", 0, "pin vLLM KV cache size on every profile") + var trims trimFlags + flags.Var(&trims, "trim", "cap a context's ladder with a reason, e.g. 64k=8:'12 GiB KV budget'; repeatable") out := flags.String("out", "", "output spec path (default stdout)") _ = flags.Parse(args) contextValues, err := parseTokenList(*contexts) @@ -99,15 +104,19 @@ func runSweepPlan(args []string) { os.Exit(2) } spec, err := sweepplan.Plan(sweepplan.PlanRequest{ - Model: *model, - Contexts: contextValues, - Concurrency: concurrencyValues, - Repeats: *repeats, - NumPrompts: *numPrompts, - PromptsPerUser: *promptsPerUser, - IncludeReference: *reference, - IncludeStress: *stress, - MinMemAvailableGiB: *memFloor, + Model: *model, + Contexts: contextValues, + Concurrency: concurrencyValues, + Repeats: *repeats, + NumPrompts: *numPrompts, + PromptsPerUser: *promptsPerUser, + IncludeReference: *reference, + IncludeStress: *stress, + MinMemAvailableGiB: *memFloor, + VLLMCommand: *vllmCommand, + GPUMemoryUtilization: *gpuMemUtil, + KVCacheMemoryBytes: *kvCacheBytes, + Trims: trims.values, }) if err != nil { fmt.Fprintln(os.Stderr, err) @@ -130,6 +139,46 @@ func runSweepPlan(args []string) { fmt.Printf("spec: %s\n", *out) } +// trimFlags parses repeatable --trim values of the form +// "=:", e.g. 64k=8:'12 GiB KV budget'. +type trimFlags struct { + values []vllmbench.LadderTrim +} + +func (flags *trimFlags) String() string { + parts := make([]string, 0, len(flags.values)) + for _, trim := range flags.values { + parts = append(parts, fmt.Sprintf("%d=%d:%s", trim.Context, trim.MaxConcurrency, trim.Reason)) + } + return strings.Join(parts, ",") +} + +func (flags *trimFlags) Set(value string) error { + context, rest, ok := strings.Cut(value, "=") + if !ok { + return fmt.Errorf("trim %q: want =:", value) + } + maxPart, reason, ok := strings.Cut(rest, ":") + if !ok || strings.TrimSpace(reason) == "" { + return fmt.Errorf("trim %q: a reason after ':' is required", value) + } + contextValues, err := parseTokenList(context) + if err != nil || len(contextValues) != 1 { + return fmt.Errorf("trim %q: want one context like 64k or 65536", value) + } + contextTokens := contextValues[0] + maxConcurrency, err := strconv.Atoi(strings.TrimSpace(maxPart)) + if err != nil || maxConcurrency <= 0 { + return fmt.Errorf("trim %q: max concurrency must be a positive integer", value) + } + flags.values = append(flags.values, vllmbench.LadderTrim{ + Context: contextTokens, + MaxConcurrency: maxConcurrency, + Reason: strings.TrimSpace(reason), + }) + return nil +} + // parseTokenList parses values such as "8k,16k,32768" into token counts. func parseTokenList(value string) ([]int, error) { var values []int @@ -260,6 +309,7 @@ func runBench(args []string) { _ = flags.Parse(args) spec := mustLoadSpec(*specPath, filterFlags.Filter()) applyOverrides(&spec, overrides) + fmt.Printf("spec provenance: %s\n", spec.Provenance) ctx := context.Background() cancel := func() {} if *timeout > 0 { @@ -717,7 +767,7 @@ func usageRoot() { localperf artifact check runs/example.sqlite localperf artifact render runs/example.sqlite [--output runs/example.html] [--store] localperf artifact merge --into runs/models/model.sqlite src1.sqlite [src2.sqlite ...] - localperf sweep plan --model model-id [--contexts 8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--out spec.json] + localperf sweep plan --model model-id [--contexts 8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--vllm-command /path/to/vllm] [--gpu-memory-utilization 0.4] [--kv-cache-memory-bytes N] [--trim 64k=8:'reason'] [--out spec.json] localperf view runs/model.sqlite [runs/other.sqlite ...] [--addr 127.0.0.1:0] [--open]`) } diff --git a/internal/report/html.go b/internal/report/html.go index 859318f..b42430f 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -49,6 +49,9 @@ type SQLiteReportDocument struct { EventCounts []SQLiteReportCount NotableEvents []SQLiteReportEvent Commands []SQLiteReportCommand + SpecProvenance string + SpecGenerator *artifact.GeneratorStamp + SpecConcurrency []int ExistingReports []SQLiteReportExport ArtifactSummaries []SQLiteReportArtifactSummary MeasurementMetrics map[int64]map[string]SQLiteReportMetric @@ -449,6 +452,7 @@ func LoadSQLiteReport(path string) (SQLiteReportDocument, error) { loadSQLiteReportEventCounts, loadSQLiteReportNotableEvents, loadSQLiteReportCommands, + loadSQLiteReportSpecProvenance, loadSQLiteReportExports, loadSQLiteReportArtifactSummaries, } { @@ -459,7 +463,7 @@ func LoadSQLiteReport(path string) (SQLiteReportDocument, error) { doc.Measurements, doc.RepeatDetails = aggregateRepeatMeasurements(doc.Measurements) doc.Legend = ReportMetrics doc.MetadataItems = sqliteReportMetadataItems(doc) - doc.ThroughputRows = sqliteReportThroughputRows(doc) + doc.ThroughputRows = append(sqliteReportThroughputRows(doc), trimmedThroughputRows(doc)...) doc.ThroughputGroups = sqliteReportThroughputGroups(doc.ThroughputRows) doc.PhaseSections = sqliteReportPhaseSections(doc.Measurements) doc.Charts = sqliteReportCharts(doc.Measurements) @@ -1346,8 +1350,107 @@ func sqliteReportCharts(measurements []SQLiteReportMeasurement) []SQLiteReportCh return []SQLiteReportChart{{Title: "Aggregate Output Throughput", Unit: "tok/s", Height: 24 + len(bars)*28, Bars: bars}} } +// loadSQLiteReportSpecProvenance verifies the latest run's original spec +// against its generator stamp: generated, edited after generation, or +// custom (no stamp). Verification happens here, from the stored bytes — +// the label is never taken on trust from the runner. +func loadSQLiteReportSpecProvenance(db *sql.DB, doc *SQLiteReportDocument) error { + var content string + err := db.QueryRow(`SELECT content FROM specs WHERE run_id = ? AND kind = 'original'`, doc.Run.ID).Scan(&content) + if err == sql.ErrNoRows { + doc.SpecProvenance = artifact.SpecProvenanceCustom + return nil + } + if err != nil { + return err + } + status, stamp := artifact.VerifySpecProvenance([]byte(content)) + doc.SpecProvenance = status + doc.SpecGenerator = stamp + if status == artifact.SpecProvenanceGenerated && stamp != nil && len(stamp.Intent) > 0 { + var intent struct { + Concurrency []int `json:"concurrency"` + } + if err := json.Unmarshal(stamp.Intent, &intent); err == nil { + doc.SpecConcurrency = intent.Concurrency + } + } + return nil +} + +func specProvenanceDisplay(doc SQLiteReportDocument) string { + switch doc.SpecProvenance { + case artifact.SpecProvenanceGenerated: + return "Generated default sweep (" + doc.SpecGenerator.Tool + ")" + case artifact.SpecProvenanceEdited: + return "Custom grid (edited after generation)" + default: + return "Custom grid (hand-authored)" + } +} + +// trimmedThroughputRows synthesizes rows for author-trimmed ladder points so +// declared trims render like adaptive skips, never as silent holes. Only +// verified generated specs are trusted for this. +func trimmedThroughputRows(doc SQLiteReportDocument) []SQLiteReportThroughputRow { + if doc.SpecProvenance != artifact.SpecProvenanceGenerated || doc.SpecGenerator == nil { + return nil + } + ladder := doc.SpecConcurrency + if len(ladder) == 0 { + return nil + } + var rows []SQLiteReportThroughputRow + for _, trim := range doc.SpecGenerator.LadderTrims { + label := contextLabel(trim.Context) + for _, concurrency := range ladder { + if concurrency <= trim.MaxConcurrency { + continue + } + for _, mode := range []string{"decode", "prefill"} { + rows = append(rows, SQLiteReportThroughputRow{ + Phase: bench.PhaseTitle(mode), + Mode: mode, + RunID: doc.Run.ID, + Profile: label, + Model: doc.Run.Name, + ContextWindow: trim.Context, + ContextLabel: "unverified (declared " + label + " active)", + ContextSortKey: trim.Context, + ContextTarget: trim.Context, + ContextSemantics: "active", + Concurrency: concurrency, + Shape: "-", + ThroughputTokS: "trimmed", + PerUserTokS: "trimmed", + TTFTMeanMS: "-", + TTFTP99MS: "-", + LatencyP95MS: "-", + Status: "skipped", + FailureLabel: "trimmed", + FailureReason: "trimmed by author: " + trim.Reason, + Detail: SQLiteReportCellDetail{ + Available: true, + Phase: bench.PhaseTitle(mode), + Mode: mode, + Status: "skipped", + FailureLabel: "trimmed", + FailureReason: "trimmed by author: " + trim.Reason, + Profile: label, + ContextLabel: "unverified (declared " + label + " active)", + ContextWindow: trim.Context, + Concurrency: concurrency, + }, + }) + } + } + } + return rows +} + func sqliteReportMetadataItems(doc SQLiteReportDocument) []SQLiteReportMetadataItem { items := []SQLiteReportMetadataItem{ + {Label: "Spec", Value: specProvenanceDisplay(doc)}, {Label: "Engine", Value: joinUnique(engineSummaries(doc.Engines), ", ")}, {Label: "Runs", Value: fmt.Sprint(len(doc.Runs))}, {Label: "Hardware", Value: bench.FirstNonEmpty(doc.Run.Hardware, "-")}, diff --git a/internal/report/html_test.go b/internal/report/html_test.go index fc4d601..ca6a6b7 100644 --- a/internal/report/html_test.go +++ b/internal/report/html_test.go @@ -2,6 +2,7 @@ package report import ( "database/sql" + "encoding/json" "os" "path/filepath" "strings" @@ -1001,3 +1002,83 @@ func TestCellDetailIncludesMetrics(t *testing.T) { } } } + +func TestSpecProvenanceRendersInReport(t *testing.T) { + artifactPath := filepath.Join(t.TempDir(), "run.sqlite") + createTestSQLiteHTMLArtifact(t, artifactPath, "Provenance") + doc, err := LoadSQLiteReport(artifactPath) + if err != nil { + t.Fatal(err) + } + // The fixture's original spec has no generator stamp. + found := false + for _, item := range doc.MetadataItems { + if item.Label == "Spec" { + found = true + if item.Value != "Custom grid (hand-authored)" { + t.Fatalf("spec provenance = %q, want custom grid for an unstamped spec", item.Value) + } + } + } + if !found { + t.Fatal("metadata missing Spec provenance item") + } +} + +func TestGeneratedSpecTrimsRenderAsRows(t *testing.T) { + artifactPath := filepath.Join(t.TempDir(), "run.sqlite") + createTestSQLiteHTMLArtifact(t, artifactPath, "Trims") + base := map[string]any{"version": "1", "name": "stamped", "model": "example/model"} + baseJSON, err := json.Marshal(base) + if err != nil { + t.Fatal(err) + } + hash, err := artifact.CanonicalSpecHash(baseJSON) + if err != nil { + t.Fatal(err) + } + base["generator"] = artifact.GeneratorStamp{ + Tool: "localperf sweep plan", + Version: "1", + Intent: json.RawMessage(`{"concurrency":[1,4,8,16]}`), + LadderTrims: []artifact.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, + ContentHash: hash, + } + content, err := json.Marshal(base) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", artifactPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE specs SET content = ? WHERE kind = 'original'`, string(content)); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + doc, err := LoadSQLiteReport(artifactPath) + if err != nil { + t.Fatal(err) + } + trimmed := []SQLiteReportThroughputRow{} + for _, row := range doc.ThroughputRows { + if row.FailureLabel == "trimmed" { + trimmed = append(trimmed, row) + } + } + // c16 above the trim cap, decode + prefill. + if len(trimmed) != 2 { + t.Fatalf("trimmed rows = %d, want 2 synthesized rows for c16", len(trimmed)) + } + row := trimmed[0] + if row.Concurrency != 16 || row.ContextTarget != 65536 || !strings.Contains(row.FailureReason, "12 GiB KV budget") { + t.Fatalf("trimmed row = %+v, want c16 at 64k with the declared reason", row) + } + for _, item := range doc.MetadataItems { + if item.Label == "Spec" && !strings.Contains(item.Value, "Generated default sweep") { + t.Fatalf("spec provenance = %q, want generated", item.Value) + } + } +} diff --git a/internal/sweepplan/provenance_test.go b/internal/sweepplan/provenance_test.go new file mode 100644 index 0000000..f10c969 --- /dev/null +++ b/internal/sweepplan/provenance_test.go @@ -0,0 +1,114 @@ +package sweepplan + +import ( + "testing" + + "github.com/dutifuldev/localperf/internal/vllmbench" +) + +func TestPlanStampsVerifiableProvenance(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{8192, 65536}, + Trims: []vllmbench.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, + }) + if err != nil { + t.Fatal(err) + } + if spec.Generator == nil || spec.Generator.Tool != "localperf sweep plan" { + t.Fatalf("generator stamp = %+v, want tool identity", spec.Generator) + } + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceGenerated { + t.Fatalf("provenance = %q, want generated for an unedited spec", got) + } + spec.Workloads[0].NumPrompts = 999 + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceEdited { + t.Fatalf("provenance = %q, want edited after mutation", got) + } + spec.Generator = nil + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceCustom { + t.Fatalf("provenance = %q, want custom without a stamp", got) + } +} + +func TestTrimsCapLaddersAndRideTheStamp(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{32768, 65536}, + Trims: []vllmbench.LadderTrim{ + {Context: 32768, MaxConcurrency: 16, Reason: "KV budget"}, + {Context: 65536, MaxConcurrency: 8, Reason: "KV budget"}, + }, + }) + if err != nil { + t.Fatal(err) + } + maxByWorkload := map[string]int{} + for _, workload := range spec.Workloads { + top := 0 + for _, value := range workload.MaxConcurrency { + if value > top { + top = value + } + } + maxByWorkload[workload.Name] = top + } + if maxByWorkload["decode-32k"] != 16 || maxByWorkload["prefill-32k"] != 16 { + t.Fatalf("32k ladders = %v, want capped at 16", maxByWorkload) + } + if maxByWorkload["decode-64k"] != 8 || maxByWorkload["prefill-64k"] != 8 { + t.Fatalf("64k ladders = %v, want capped at 8", maxByWorkload) + } + if len(spec.Generator.LadderTrims) != 2 { + t.Fatalf("stamped trims = %+v, want both declared trims", spec.Generator.LadderTrims) + } + for _, profile := range spec.Profiles { + if profile.Name == "64k" && profile.MaxNumSeqs != 8 { + t.Fatalf("64k max_num_seqs = %d, want sized to trimmed ladder", profile.MaxNumSeqs) + } + } +} + +func TestRuntimeIntentReachesSpec(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{8192}, + VLLMCommand: "/opt/runtimes/vllm/bin/vllm", + GPUMemoryUtilization: 0.4, + KVCacheMemoryBytes: 12 << 30, + }) + if err != nil { + t.Fatal(err) + } + if spec.Runner.VLLMCommand != "/opt/runtimes/vllm/bin/vllm" { + t.Fatalf("vllm command = %q, want intent runtime path", spec.Runner.VLLMCommand) + } + for _, profile := range spec.Profiles { + if profile.GPUMemoryUtilization != 0.4 { + t.Fatalf("profile %s gpu mem = %v, want 0.4", profile.Name, profile.GPUMemoryUtilization) + } + found := false + for index, arg := range profile.Args { + if arg == "--kv-cache-memory-bytes" && index+1 < len(profile.Args) && profile.Args[index+1] == "12884901888" { + found = true + } + } + if !found { + t.Fatalf("profile %s args = %v, want kv cache bytes pinned", profile.Name, profile.Args) + } + } + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceGenerated { + t.Fatalf("provenance = %q, want generated", got) + } +} + +func TestTrimWithoutReasonIsRejected(t *testing.T) { + _, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{65536}, + Trims: []vllmbench.LadderTrim{{Context: 65536, MaxConcurrency: 8}}, + }) + if err == nil { + t.Fatal("expected a validation error for a trim without a reason") + } +} diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index adef17a..b4315e1 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -8,7 +8,9 @@ package sweepplan import ( + "encoding/json" "fmt" + "strconv" "strings" "github.com/dutifuldev/localperf/internal/vllmbench" @@ -16,29 +18,41 @@ import ( type PlanRequest struct { // Model is the model identifier to benchmark (required). - Model string + Model string `json:"model"` // Contexts is the active-context ladder in tokens, e.g. 8192..131072. - Contexts []int + Contexts []int `json:"contexts,omitempty"` // Concurrency levels per point, e.g. 1,4,8,16,32. - Concurrency []int + Concurrency []int `json:"concurrency,omitempty"` // Repeats per measurement; defaults to 1. - Repeats int + Repeats int `json:"repeats,omitempty"` // NumPrompts fixes the request count per measurement; when 0, prompts // scale with concurrency via PromptsPerUser instead. - NumPrompts int + NumPrompts int `json:"num_prompts,omitempty"` // PromptsPerUser scales requests with concurrency // (num_prompts = max(8, PromptsPerUser * concurrency)); defaults to 2. - PromptsPerUser int + PromptsPerUser int `json:"prompts_per_user,omitempty"` // IncludeReference adds the 4k max-throughput-reference capacity family. - IncludeReference bool + IncludeReference bool `json:"include_reference,omitempty"` // IncludeStress adds long-output decode spot checks (4096 tokens at // 32k c4 and 64k c1/c4 when those contexts are in the ladder) and the // 128k points at c1/c4. - IncludeStress bool + IncludeStress bool `json:"include_stress,omitempty"` // MinMemAvailableGiB is the safety memory floor; defaults to 40. - MinMemAvailableGiB float64 + MinMemAvailableGiB float64 `json:"min_mem_available_gib,omitempty"` // BasePort is the first server port; defaults to 8100. - BasePort int + BasePort int `json:"base_port,omitempty"` + // VLLMCommand overrides the vllm executable for managed serves — the + // machine-specific runtime path that previously forced hand-editing. + VLLMCommand string `json:"vllm_command,omitempty"` + // GPUMemoryUtilization applies to every generated profile when set. + GPUMemoryUtilization float64 `json:"gpu_memory_utilization,omitempty"` + // KVCacheMemoryBytes pins vLLM's KV cache size on every profile via + // --kv-cache-memory-bytes when set. + KVCacheMemoryBytes int64 `json:"kv_cache_memory_bytes,omitempty"` + // Trims are declared author decisions to cap a context's concurrency + // ladder; each needs a reason and renders in reports like an adaptive + // skip, never as a silent hole. + Trims []vllmbench.LadderTrim `json:"trims,omitempty"` } const ( @@ -102,6 +116,14 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { if len(request.Contexts) == 0 && !request.IncludeReference { return vllmbench.Spec{}, fmt.Errorf("at least one context point or the reference family is required") } + for index, trim := range request.Trims { + if trim.Context <= 0 || trim.MaxConcurrency <= 0 { + return vllmbench.Spec{}, fmt.Errorf("trims[%d]: context and max_concurrency must be positive", index) + } + if strings.TrimSpace(trim.Reason) == "" { + return vllmbench.Spec{}, fmt.Errorf("trims[%d]: a reason is required — declared trims render in reports", index) + } + } if len(request.Concurrency) == 0 { request.Concurrency = []int{1, 4, 8, 16, 32} } @@ -173,15 +195,67 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { spec.Workloads = append(spec.Workloads, stressWorkloads(contexts, request)...) } + applyRuntimeIntent(&spec, request) + // Emit the normalized spec: explicit defaults are stable to diff and // leave nothing for a reader to guess. vllmbench.ApplyDefaults(&spec) if err := vllmbench.ValidateSpec(spec); err != nil { return vllmbench.Spec{}, fmt.Errorf("generated spec failed validation (generator/validator drift): %w", err) } + if err := stampSpec(&spec, request); err != nil { + return vllmbench.Spec{}, err + } return spec, nil } +// applyRuntimeIntent carries machine-specific runtime choices into the spec +// so nothing forces hand-editing the generated file. +func applyRuntimeIntent(spec *vllmbench.Spec, request PlanRequest) { + if strings.TrimSpace(request.VLLMCommand) != "" { + spec.Runner.VLLMCommand = request.VLLMCommand + } + for index := range spec.Profiles { + if request.GPUMemoryUtilization > 0 { + spec.Profiles[index].GPUMemoryUtilization = request.GPUMemoryUtilization + } + if request.KVCacheMemoryBytes > 0 { + spec.Profiles[index].Args = append(spec.Profiles[index].Args, + "--kv-cache-memory-bytes", strconv.FormatInt(request.KVCacheMemoryBytes, 10)) + } + } +} + +// generatorTool and generatorVersion identify the stamp format, not the +// binary build: the content hash is what verification relies on. +const ( + generatorTool = "localperf sweep plan" + generatorVersion = "1" +) + +// stampSpec writes the provenance record: the intent that produced the +// spec, declared ladder trims, and a content hash over the spec with the +// stamp removed. `bench run` and reports verify the hash; any edit after +// generation demotes the spec to a custom grid. +func stampSpec(spec *vllmbench.Spec, request PlanRequest) error { + intent, err := json.Marshal(request) + if err != nil { + return err + } + spec.Generator = &vllmbench.GeneratorStamp{ + Tool: generatorTool, + Version: generatorVersion, + Intent: intent, + LadderTrims: append([]vllmbench.LadderTrim{}, request.Trims...), + } + hash, err := vllmbench.SpecContentHash(*spec) + if err != nil { + return err + } + spec.Generator.ContentHash = hash + return nil +} + func sweepProfile(name string, maxModelLen, port, maxNumSeqs int) vllmbench.Profile { return vllmbench.Profile{ Name: name, @@ -226,7 +300,7 @@ func stressWorkloads(contexts []int, request PlanRequest) []vllmbench.Workload { // ladder plus any stress spot checks attached to that context; a stress c4 // point must not run against a server that only admits c1. func contextMaxSeqs(context int, request PlanRequest) int { - seqs := maxConcurrencyOf(pointConcurrency(context, request.Concurrency)) + seqs := maxConcurrencyOf(pointConcurrency(context, request.Concurrency, request.Trims)) if request.IncludeStress { for _, spot := range stressSpots { if spot.context == context { @@ -258,13 +332,22 @@ func containsInt(values []int, wanted int) bool { // pointConcurrency caps contexts >= 128k at c4; higher concurrency at those // KV budgets is stress territory, not the default grid. -func pointConcurrency(context int, concurrency []int) []int { - if context < stressContext { +func pointConcurrency(context int, concurrency []int, trims []vllmbench.LadderTrim) []int { + cap := 0 + if context >= stressContext { + cap = highContextConcurrencyCap + } + for _, trim := range trims { + if trim.Context == context && (cap == 0 || trim.MaxConcurrency < cap) { + cap = trim.MaxConcurrency + } + } + if cap == 0 { return append([]int{}, concurrency...) } capped := []int{} for _, value := range concurrency { - if value <= highContextConcurrencyCap { + if value <= cap { capped = append(capped, value) } } @@ -294,7 +377,7 @@ func sweepWorkload(name, phase, profile string, target int, semantics string, in }, NumPrompts: request.NumPrompts, PromptsPerUser: request.PromptsPerUser, - MaxConcurrency: pointConcurrency(target, request.Concurrency), + MaxConcurrency: pointConcurrency(target, request.Concurrency, request.Trims), IgnoreEOS: true, Temperature: &temperature, } diff --git a/internal/sweepplan/testdata/default-sweep-8k.golden.json b/internal/sweepplan/testdata/default-sweep-8k.golden.json index 9dd3b8d..02a703a 100644 --- a/internal/sweepplan/testdata/default-sweep-8k.golden.json +++ b/internal/sweepplan/testdata/default-sweep-8k.golden.json @@ -168,5 +168,24 @@ "ignore_eos": true, "temperature": 0 } - ] + ], + "generator": { + "tool": "localperf sweep plan", + "version": "1", + "intent": { + "model": "example/model", + "contexts": [ + 8192 + ], + "concurrency": [ + 1, + 4 + ], + "prompts_per_user": 2, + "include_reference": true, + "min_mem_available_gib": 40, + "base_port": 8100 + }, + "content_hash": "68a8c6112714a62f403867cd8e32344d83f2f52697e048deb5def9d7240d3bd3" + } } diff --git a/internal/vllmbench/config.go b/internal/vllmbench/config.go index 1d12bdb..88776eb 100644 --- a/internal/vllmbench/config.go +++ b/internal/vllmbench/config.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "github.com/dutifuldev/localperf/internal/artifact" + "github.com/dutifuldev/localperf/internal/bench" "github.com/dutifuldev/localperf/internal/collections" ) @@ -40,18 +42,58 @@ const ( ) type Spec struct { - Version string `json:"version"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - Model string `json:"model"` - OutputDir string `json:"output_dir,omitempty"` - Env map[string]string `json:"env,omitempty"` - Engines []EngineConfig `json:"engines,omitempty"` - Runner RunnerConfig `json:"runner"` - Safety SafetyConfig `json:"safety"` - Warmup WarmupConfig `json:"warmup,omitempty"` - Profiles []Profile `json:"profiles"` - Workloads []Workload `json:"workloads"` + Version string `json:"version"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Model string `json:"model"` + OutputDir string `json:"output_dir,omitempty"` + Env map[string]string `json:"env,omitempty"` + Engines []EngineConfig `json:"engines,omitempty"` + Runner RunnerConfig `json:"runner"` + Safety SafetyConfig `json:"safety"` + Warmup WarmupConfig `json:"warmup,omitempty"` + Profiles []Profile `json:"profiles"` + Workloads []Workload `json:"workloads"` + Generator *artifact.GeneratorStamp `json:"generator,omitempty"` + + // Provenance is computed at load time from the generator stamp; it is + // never serialized or taken on trust. + Provenance string `json:"-"` +} + +// SpecContentHash hashes the canonical spec document with the generator +// stamp removed; see artifact.CanonicalSpecHash. +func SpecContentHash(spec Spec) (string, error) { + data, err := json.Marshal(spec) + if err != nil { + return "", err + } + return artifact.CanonicalSpecHash(data) +} + +// Provenance types and statuses re-exported for callers of this package, +// so higher layers do not need to import the artifact format directly. +type ( + GeneratorStamp = artifact.GeneratorStamp + LadderTrim = artifact.LadderTrim +) + +const ( + SpecProvenanceGenerated = artifact.SpecProvenanceGenerated + SpecProvenanceEdited = artifact.SpecProvenanceEdited + SpecProvenanceCustom = artifact.SpecProvenanceCustom +) + +// SpecProvenance verifies a spec's generator stamp: "generated" when the +// content hash matches, "edited" when a stamp exists but the content +// changed after generation, "custom" when there is no stamp. +func SpecProvenance(spec Spec) string { + data, err := json.Marshal(spec) + if err != nil { + return artifact.SpecProvenanceCustom + } + status, _ := artifact.VerifySpecProvenance(data) + return status } type EngineConfig struct { @@ -238,6 +280,9 @@ func LoadSpec(path string) (Spec, error) { if err := json.Unmarshal(data, &spec); err != nil { return Spec{}, err } + // Verify provenance against the raw bytes, before defaults mutate the + // spec away from what the generator hashed. + spec.Provenance = SpecProvenance(spec) ApplyDefaults(&spec) if err := ValidateSpec(spec); err != nil { return Spec{}, err @@ -681,12 +726,40 @@ func ValidateSpec(spec Spec) error { issues = append(issues, validateEndpointBaseURLProfileUsage(spec)...) issues = append(issues, validateWarmup(spec.Warmup)...) issues = append(issues, validateWorkloads(spec.Workloads, profileNames, spec.Profiles)...) + issues = append(issues, validateGeneratorStamp(spec.Generator)...) if len(issues) > 0 { return errors.New(strings.Join(issues, "\n")) } return nil } +// validateGeneratorStamp requires every declared ladder trim to carry its +// reason: an unexplained trim is a silent hole waiting to render. +func validateGeneratorStamp(stamp *artifact.GeneratorStamp) []string { + if stamp == nil { + return nil + } + var issues []string + for index, trim := range stamp.LadderTrims { + issues = append(issues, validateLadderTrim(index, trim)...) + } + return issues +} + +func validateLadderTrim(index int, trim artifact.LadderTrim) []string { + var issues []string + if trim.Context <= 0 { + issues = append(issues, fmt.Sprintf("generator.ladder_trims[%d]: context must be positive", index)) + } + if trim.MaxConcurrency <= 0 { + issues = append(issues, fmt.Sprintf("generator.ladder_trims[%d]: max_concurrency must be positive", index)) + } + if strings.TrimSpace(trim.Reason) == "" { + issues = append(issues, fmt.Sprintf("generator.ladder_trims[%d]: reason is required", index)) + } + return issues +} + func validateSpecBasics(spec Spec) []string { var issues []string if strings.TrimSpace(spec.Name) == "" { diff --git a/internal/vllmbench/config_test.go b/internal/vllmbench/config_test.go index 5b853cd..7e7027b 100644 --- a/internal/vllmbench/config_test.go +++ b/internal/vllmbench/config_test.go @@ -3671,3 +3671,19 @@ func TestPrefixCachingResolvedFromArgs(t *testing.T) { t.Fatalf("prefix caching = %v, want unknown without any signal", got) } } + +func TestValidateLadderTrimBranches(t *testing.T) { + if issues := validateGeneratorStamp(nil); len(issues) != 0 { + t.Fatalf("nil stamp issues = %v, want none", issues) + } + stamp := &GeneratorStamp{LadderTrims: []LadderTrim{ + {Context: 0, MaxConcurrency: 8, Reason: "r"}, + {Context: 65536, MaxConcurrency: 0, Reason: "r"}, + {Context: 65536, MaxConcurrency: 8, Reason: " "}, + {Context: 65536, MaxConcurrency: 8, Reason: "ok"}, + }} + issues := validateGeneratorStamp(stamp) + if len(issues) != 3 { + t.Fatalf("issues = %v, want one per invalid trim", issues) + } +} From be6af8f44f7b4f8880c40d46c188c52db853577a Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 16:36:17 +0800 Subject: [PATCH 2/3] fix(spec): hash covers stamp metadata; trims validated and never shadow real rows Codex review round 1: the content hash now excludes only generator.content_hash itself, so tampering with declared trims or intent demotes the spec; bench run verifies provenance from the raw file bytes; trims must reference generated contexts; synthesized trimmed rows never overwrite real measurements. --- internal/artifact/provenance.go | 14 +++++++----- internal/report/html.go | 14 ++++++++++-- internal/report/html_test.go | 22 +++++++++++-------- internal/sweepplan/provenance_test.go | 14 ++++++++++++ internal/sweepplan/sweepplan.go | 3 +++ .../testdata/default-sweep-8k.golden.json | 2 +- internal/vllmbench/config.go | 7 +++--- 7 files changed, 56 insertions(+), 20 deletions(-) diff --git a/internal/artifact/provenance.go b/internal/artifact/provenance.go index b050920..38ca354 100644 --- a/internal/artifact/provenance.go +++ b/internal/artifact/provenance.go @@ -35,16 +35,20 @@ const ( SpecProvenanceCustom = "custom" ) -// CanonicalSpecHash hashes a spec document with the "generator" key removed, -// after canonicalizing through a generic JSON map (sorted keys), so the -// generator and every verifier agree byte-for-byte regardless of struct -// field order. +// CanonicalSpecHash hashes a spec document with only the self-referential +// generator.content_hash removed, after canonicalizing through a generic +// JSON map (sorted keys), so the generator and every verifier agree +// byte-for-byte regardless of struct field order. The rest of the stamp — +// intent and ladder trims — is covered by the hash: reports trust those +// fields, so editing them must demote the spec. func CanonicalSpecHash(raw []byte) (string, error) { var doc map[string]any if err := json.Unmarshal(raw, &doc); err != nil { return "", err } - delete(doc, "generator") + if generator, ok := doc["generator"].(map[string]any); ok { + delete(generator, "content_hash") + } canonical, err := json.Marshal(doc) if err != nil { return "", err diff --git a/internal/report/html.go b/internal/report/html.go index b42430f..19cf039 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -463,7 +463,8 @@ func LoadSQLiteReport(path string) (SQLiteReportDocument, error) { doc.Measurements, doc.RepeatDetails = aggregateRepeatMeasurements(doc.Measurements) doc.Legend = ReportMetrics doc.MetadataItems = sqliteReportMetadataItems(doc) - doc.ThroughputRows = append(sqliteReportThroughputRows(doc), trimmedThroughputRows(doc)...) + realRows := sqliteReportThroughputRows(doc) + doc.ThroughputRows = append(realRows, trimmedThroughputRows(doc, realRows)...) doc.ThroughputGroups = sqliteReportThroughputGroups(doc.ThroughputRows) doc.PhaseSections = sqliteReportPhaseSections(doc.Measurements) doc.Charts = sqliteReportCharts(doc.Measurements) @@ -1392,7 +1393,7 @@ func specProvenanceDisplay(doc SQLiteReportDocument) string { // trimmedThroughputRows synthesizes rows for author-trimmed ladder points so // declared trims render like adaptive skips, never as silent holes. Only // verified generated specs are trusted for this. -func trimmedThroughputRows(doc SQLiteReportDocument) []SQLiteReportThroughputRow { +func trimmedThroughputRows(doc SQLiteReportDocument, existing []SQLiteReportThroughputRow) []SQLiteReportThroughputRow { if doc.SpecProvenance != artifact.SpecProvenanceGenerated || doc.SpecGenerator == nil { return nil } @@ -1400,6 +1401,12 @@ func trimmedThroughputRows(doc SQLiteReportDocument) []SQLiteReportThroughputRow if len(ladder) == 0 { return nil } + // A real measurement — from any run in a model-level artifact — always + // wins over a synthesized trim marker for the same point. + measured := map[string]struct{}{} + for _, row := range existing { + measured[fmt.Sprintf("%d/%s/%d", row.ContextTarget, row.Mode, row.Concurrency)] = struct{}{} + } var rows []SQLiteReportThroughputRow for _, trim := range doc.SpecGenerator.LadderTrims { label := contextLabel(trim.Context) @@ -1408,6 +1415,9 @@ func trimmedThroughputRows(doc SQLiteReportDocument) []SQLiteReportThroughputRow continue } for _, mode := range []string{"decode", "prefill"} { + if _, ok := measured[fmt.Sprintf("%d/%s/%d", trim.Context, mode, concurrency)]; ok { + continue + } rows = append(rows, SQLiteReportThroughputRow{ Phase: bench.PhaseTitle(mode), Mode: mode, diff --git a/internal/report/html_test.go b/internal/report/html_test.go index ca6a6b7..ed78b3f 100644 --- a/internal/report/html_test.go +++ b/internal/report/html_test.go @@ -1029,21 +1029,25 @@ func TestGeneratedSpecTrimsRenderAsRows(t *testing.T) { artifactPath := filepath.Join(t.TempDir(), "run.sqlite") createTestSQLiteHTMLArtifact(t, artifactPath, "Trims") base := map[string]any{"version": "1", "name": "stamped", "model": "example/model"} - baseJSON, err := json.Marshal(base) + stamp := artifact.GeneratorStamp{ + Tool: "localperf sweep plan", + Version: "1", + Intent: json.RawMessage(`{"concurrency":[1,4,8,16]}`), + LadderTrims: []artifact.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, + } + base["generator"] = stamp + unhashed, err := json.Marshal(base) if err != nil { t.Fatal(err) } - hash, err := artifact.CanonicalSpecHash(baseJSON) + // The hash covers the stamp's trusted fields; only content_hash itself + // is excluded. + hash, err := artifact.CanonicalSpecHash(unhashed) if err != nil { t.Fatal(err) } - base["generator"] = artifact.GeneratorStamp{ - Tool: "localperf sweep plan", - Version: "1", - Intent: json.RawMessage(`{"concurrency":[1,4,8,16]}`), - LadderTrims: []artifact.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, - ContentHash: hash, - } + stamp.ContentHash = hash + base["generator"] = stamp content, err := json.Marshal(base) if err != nil { t.Fatal(err) diff --git a/internal/sweepplan/provenance_test.go b/internal/sweepplan/provenance_test.go index f10c969..8f915ad 100644 --- a/internal/sweepplan/provenance_test.go +++ b/internal/sweepplan/provenance_test.go @@ -25,6 +25,20 @@ func TestPlanStampsVerifiableProvenance(t *testing.T) { if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceEdited { t.Fatalf("provenance = %q, want edited after mutation", got) } + // The stamp's trusted fields are covered by the hash: reports rely on + // declared trims, so editing them must demote the spec too. + fresh, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{8192, 65536}, + Trims: []vllmbench.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, + }) + if err != nil { + t.Fatal(err) + } + fresh.Generator.LadderTrims = append(fresh.Generator.LadderTrims, vllmbench.LadderTrim{Context: 8192, MaxConcurrency: 1, Reason: "invented"}) + if got := vllmbench.SpecProvenance(fresh); got != vllmbench.SpecProvenanceEdited { + t.Fatalf("provenance = %q, want edited after tampering with declared trims", got) + } spec.Generator = nil if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceCustom { t.Fatalf("provenance = %q, want custom without a stamp", got) diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index b4315e1..a2eb171 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -123,6 +123,9 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { if strings.TrimSpace(trim.Reason) == "" { return vllmbench.Spec{}, fmt.Errorf("trims[%d]: a reason is required — declared trims render in reports", index) } + if !containsInt(request.Contexts, trim.Context) && !(request.IncludeStress && trim.Context == stressContext) { + return vllmbench.Spec{}, fmt.Errorf("trims[%d]: context %d is not in the generated ladder", index, trim.Context) + } } if len(request.Concurrency) == 0 { request.Concurrency = []int{1, 4, 8, 16, 32} diff --git a/internal/sweepplan/testdata/default-sweep-8k.golden.json b/internal/sweepplan/testdata/default-sweep-8k.golden.json index 02a703a..743ec98 100644 --- a/internal/sweepplan/testdata/default-sweep-8k.golden.json +++ b/internal/sweepplan/testdata/default-sweep-8k.golden.json @@ -186,6 +186,6 @@ "min_mem_available_gib": 40, "base_port": 8100 }, - "content_hash": "68a8c6112714a62f403867cd8e32344d83f2f52697e048deb5def9d7240d3bd3" + "content_hash": "4f4964d8f7673921cc1f398579f70cf856e0adca8487cb2516926ea6ef410cac" } } diff --git a/internal/vllmbench/config.go b/internal/vllmbench/config.go index 88776eb..8b21e15 100644 --- a/internal/vllmbench/config.go +++ b/internal/vllmbench/config.go @@ -280,9 +280,10 @@ func LoadSpec(path string) (Spec, error) { if err := json.Unmarshal(data, &spec); err != nil { return Spec{}, err } - // Verify provenance against the raw bytes, before defaults mutate the - // spec away from what the generator hashed. - spec.Provenance = SpecProvenance(spec) + // Verify provenance against the raw file bytes, before unmarshalling + // drops unknown fields and defaults mutate the spec: the runner's label + // must match what reports later verify from the stored bytes. + spec.Provenance, _ = artifact.VerifySpecProvenance(data) ApplyDefaults(&spec) if err := ValidateSpec(spec); err != nil { return Spec{}, err From cf9539a5a54ccedb1883fda0d410e2016fac565f Mon Sep 17 00:00:00 2001 From: Bob Date: Sun, 5 Jul 2026 16:44:54 +0800 Subject: [PATCH 3/3] fix(sweep): point the bench loader at the requested vllm runtime --- internal/sweepplan/provenance_test.go | 3 +++ internal/sweepplan/sweepplan.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/internal/sweepplan/provenance_test.go b/internal/sweepplan/provenance_test.go index 8f915ad..748f4b1 100644 --- a/internal/sweepplan/provenance_test.go +++ b/internal/sweepplan/provenance_test.go @@ -97,6 +97,9 @@ func TestRuntimeIntentReachesSpec(t *testing.T) { if spec.Runner.VLLMCommand != "/opt/runtimes/vllm/bin/vllm" { t.Fatalf("vllm command = %q, want intent runtime path", spec.Runner.VLLMCommand) } + if spec.Runner.VLLMBenchCommand != "/opt/runtimes/vllm/bin/vllm" { + t.Fatalf("vllm bench command = %q, want the same runtime as serve", spec.Runner.VLLMBenchCommand) + } for _, profile := range spec.Profiles { if profile.GPUMemoryUtilization != 0.4 { t.Fatalf("profile %s gpu mem = %v, want 0.4", profile.Name, profile.GPUMemoryUtilization) diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index a2eb171..d48d2fb 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -217,6 +217,10 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { func applyRuntimeIntent(spec *vllmbench.Spec, request PlanRequest) { if strings.TrimSpace(request.VLLMCommand) != "" { spec.Runner.VLLMCommand = request.VLLMCommand + // The bench loader must run from the same runtime, or the sweep + // would serve with the requested vLLM and benchmark with whatever + // is on PATH. + spec.Runner.VLLMBenchCommand = request.VLLMCommand } for index := range spec.Profiles { if request.GPUMemoryUtilization > 0 {