diff --git a/README.md b/README.md index 4873363..391c928 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,16 @@ collides with different provenance is refused instead of silently replaced. The report lists every run and aggregates repeated points across runs with mean ± spread. +## Spec Provenance + +Create specs with `sweep plan`; machine-specific runtime choices are flags +(`--vllm-command`, `--gpu-memory-utilization`, `--kv-cache-memory-bytes`), +and deliberate ladder caps are declared trims with reasons +(`--trim 64k=8:'12 GiB KV budget'`) that render in reports like adaptive +skips. Generated specs carry a verified `generator` stamp: reports label +runs "Generated default sweep" only while the spec's content hash still +matches, and anything hand-written or edited shows as "Custom grid". + ## Context Semantics Every workload must declare what its context number means: diff --git a/docs/2026-07-02-default-inference-sweep.md b/docs/2026-07-02-default-inference-sweep.md index f23de7c..41598b8 100644 --- a/docs/2026-07-02-default-inference-sweep.md +++ b/docs/2026-07-02-default-inference-sweep.md @@ -181,3 +181,32 @@ whose results already completed. Do not call a sweep complete until all completed, skipped, and failed cases are recorded with enough metadata to reproduce the run. + +## Spec Provenance + +Create sweep specs only through the generator; never hand-write or +post-edit the JSON: + +```sh +localperf sweep plan \ + --model \ + --vllm-command /path/to/runtime/bin/vllm \ + --gpu-memory-utilization 0.4 \ + --kv-cache-memory-bytes 12884901888 \ + --trim 64k=8:'12 GiB KV budget' \ + --out spec.json +``` + +Machine-specific runtime choices (vllm path, GPU memory, KV budget, extra +serve args) are generator flags, so nothing forces editing the output. A +deliberate concurrency cap is a declared trim: `--trim =:` +removes the higher points from the grid, records the decision in the spec, +and reports render the trimmed points like adaptive skips — with the reason, +never as silent holes. + +Generated specs carry a `generator` stamp (tool, intent, content hash). +`bench run` verifies the hash and prints the provenance; reports and the +viewer label the run "Generated default sweep" only when the hash still +matches. A spec without a stamp, or edited after generation, runs fine but +is labeled "Custom grid" — the label is verified from the stored spec bytes, +never taken on trust. diff --git a/docs/2026-07-05-report-integrity-plan.md b/docs/2026-07-05-report-integrity-plan.md index 2c30cf8..7898ffd 100644 --- a/docs/2026-07-05-report-integrity-plan.md +++ b/docs/2026-07-05-report-integrity-plan.md @@ -6,8 +6,8 @@ date: 2026-07-05 # Report Integrity Plan -Date: 2026-07-05. Status: PR 1 (streaming TTFT) and PR 2 (rendering + -legacy cutover) implemented; PR 3 (provenance) in progress. +Date: 2026-07-05. Status: implemented (PR 1 streaming TTFT, PR 2 +rendering + legacy cutover, PR 3 spec provenance). Fixes every problem found while inspecting the live viewer serving the 4-model GB10 sweep of 2026-07-05, each verified against the code. diff --git a/internal/artifact/provenance.go b/internal/artifact/provenance.go new file mode 100644 index 0000000..38ca354 --- /dev/null +++ b/internal/artifact/provenance.go @@ -0,0 +1,76 @@ +package artifact + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +// GeneratorStamp is the provenance record `sweep plan` writes into generated +// specs. ContentHash covers the canonical spec document with the "generator" +// key removed, so any post-generation edit is detectable; see +// docs/2026-07-05-report-integrity-plan.md. +type GeneratorStamp struct { + Tool string `json:"tool"` + Version string `json:"version"` + Intent json.RawMessage `json:"intent,omitempty"` + LadderTrims []LadderTrim `json:"ladder_trims,omitempty"` + ContentHash string `json:"content_hash"` +} + +// LadderTrim declares an author decision to cap a context's concurrency +// ladder, with the reason rendered in reports so trimmed points never look +// like silent holes. +type LadderTrim struct { + Context int `json:"context"` + MaxConcurrency int `json:"max_concurrency"` + Reason string `json:"reason"` +} + +// Spec provenance statuses; reports label anything but a verified generated +// spec as a custom grid. +const ( + SpecProvenanceGenerated = "generated" + SpecProvenanceEdited = "edited" + SpecProvenanceCustom = "custom" +) + +// CanonicalSpecHash hashes a spec document with only the self-referential +// generator.content_hash removed, after canonicalizing through a generic +// JSON map (sorted keys), so the generator and every verifier agree +// byte-for-byte regardless of struct field order. The rest of the stamp — +// intent and ladder trims — is covered by the hash: reports trust those +// fields, so editing them must demote the spec. +func CanonicalSpecHash(raw []byte) (string, error) { + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + return "", err + } + if generator, ok := doc["generator"].(map[string]any); ok { + delete(generator, "content_hash") + } + canonical, err := json.Marshal(doc) + if err != nil { + return "", err + } + sum := sha256.Sum256(canonical) + return hex.EncodeToString(sum[:]), nil +} + +// VerifySpecProvenance checks raw spec bytes against their embedded +// generator stamp: "generated" when the content hash matches, "edited" when +// a stamp exists but the content changed, "custom" when there is no stamp +// or the document does not parse. +func VerifySpecProvenance(raw []byte) (string, *GeneratorStamp) { + var doc struct { + Generator *GeneratorStamp `json:"generator"` + } + if err := json.Unmarshal(raw, &doc); err != nil || doc.Generator == nil { + return SpecProvenanceCustom, nil + } + hash, err := CanonicalSpecHash(raw) + if err != nil || hash != doc.Generator.ContentHash { + return SpecProvenanceEdited, doc.Generator + } + return SpecProvenanceGenerated, doc.Generator +} diff --git a/internal/benchcli/benchcli.go b/internal/benchcli/benchcli.go index 24f6d6c..cb72656 100644 --- a/internal/benchcli/benchcli.go +++ b/internal/benchcli/benchcli.go @@ -86,6 +86,11 @@ func runSweepPlan(args []string) { reference := flags.Bool("reference", true, "include the 4k max-throughput-reference capacity family") stress := flags.Bool("stress", false, "add long-output decode spot checks (4096 tokens at 32k c4, 64k c1/c4) and the 128k points") memFloor := flags.Float64("min-mem-available-gib", 0, "safety memory floor in GiB (default 40)") + vllmCommand := flags.String("vllm-command", "", "vllm executable for managed serves (machine-specific runtime path)") + gpuMemUtil := flags.Float64("gpu-memory-utilization", 0, "gpu memory utilization applied to every profile") + kvCacheBytes := flags.Int64("kv-cache-memory-bytes", 0, "pin vLLM KV cache size on every profile") + var trims trimFlags + flags.Var(&trims, "trim", "cap a context's ladder with a reason, e.g. 64k=8:'12 GiB KV budget'; repeatable") out := flags.String("out", "", "output spec path (default stdout)") _ = flags.Parse(args) contextValues, err := parseTokenList(*contexts) @@ -99,15 +104,19 @@ func runSweepPlan(args []string) { os.Exit(2) } spec, err := sweepplan.Plan(sweepplan.PlanRequest{ - Model: *model, - Contexts: contextValues, - Concurrency: concurrencyValues, - Repeats: *repeats, - NumPrompts: *numPrompts, - PromptsPerUser: *promptsPerUser, - IncludeReference: *reference, - IncludeStress: *stress, - MinMemAvailableGiB: *memFloor, + Model: *model, + Contexts: contextValues, + Concurrency: concurrencyValues, + Repeats: *repeats, + NumPrompts: *numPrompts, + PromptsPerUser: *promptsPerUser, + IncludeReference: *reference, + IncludeStress: *stress, + MinMemAvailableGiB: *memFloor, + VLLMCommand: *vllmCommand, + GPUMemoryUtilization: *gpuMemUtil, + KVCacheMemoryBytes: *kvCacheBytes, + Trims: trims.values, }) if err != nil { fmt.Fprintln(os.Stderr, err) @@ -130,6 +139,46 @@ func runSweepPlan(args []string) { fmt.Printf("spec: %s\n", *out) } +// trimFlags parses repeatable --trim values of the form +// "=:", e.g. 64k=8:'12 GiB KV budget'. +type trimFlags struct { + values []vllmbench.LadderTrim +} + +func (flags *trimFlags) String() string { + parts := make([]string, 0, len(flags.values)) + for _, trim := range flags.values { + parts = append(parts, fmt.Sprintf("%d=%d:%s", trim.Context, trim.MaxConcurrency, trim.Reason)) + } + return strings.Join(parts, ",") +} + +func (flags *trimFlags) Set(value string) error { + context, rest, ok := strings.Cut(value, "=") + if !ok { + return fmt.Errorf("trim %q: want =:", value) + } + maxPart, reason, ok := strings.Cut(rest, ":") + if !ok || strings.TrimSpace(reason) == "" { + return fmt.Errorf("trim %q: a reason after ':' is required", value) + } + contextValues, err := parseTokenList(context) + if err != nil || len(contextValues) != 1 { + return fmt.Errorf("trim %q: want one context like 64k or 65536", value) + } + contextTokens := contextValues[0] + maxConcurrency, err := strconv.Atoi(strings.TrimSpace(maxPart)) + if err != nil || maxConcurrency <= 0 { + return fmt.Errorf("trim %q: max concurrency must be a positive integer", value) + } + flags.values = append(flags.values, vllmbench.LadderTrim{ + Context: contextTokens, + MaxConcurrency: maxConcurrency, + Reason: strings.TrimSpace(reason), + }) + return nil +} + // parseTokenList parses values such as "8k,16k,32768" into token counts. func parseTokenList(value string) ([]int, error) { var values []int @@ -260,6 +309,7 @@ func runBench(args []string) { _ = flags.Parse(args) spec := mustLoadSpec(*specPath, filterFlags.Filter()) applyOverrides(&spec, overrides) + fmt.Printf("spec provenance: %s\n", spec.Provenance) ctx := context.Background() cancel := func() {} if *timeout > 0 { @@ -717,7 +767,7 @@ func usageRoot() { localperf artifact check runs/example.sqlite localperf artifact render runs/example.sqlite [--output runs/example.html] [--store] localperf artifact merge --into runs/models/model.sqlite src1.sqlite [src2.sqlite ...] - localperf sweep plan --model model-id [--contexts 8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--out spec.json] + localperf sweep plan --model model-id [--contexts 8k,16k,32k,64k] [--concurrency 1,4,8,16,32] [--repeats 1] [--reference] [--stress] [--vllm-command /path/to/vllm] [--gpu-memory-utilization 0.4] [--kv-cache-memory-bytes N] [--trim 64k=8:'reason'] [--out spec.json] localperf view runs/model.sqlite [runs/other.sqlite ...] [--addr 127.0.0.1:0] [--open]`) } diff --git a/internal/report/html.go b/internal/report/html.go index 859318f..19cf039 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -49,6 +49,9 @@ type SQLiteReportDocument struct { EventCounts []SQLiteReportCount NotableEvents []SQLiteReportEvent Commands []SQLiteReportCommand + SpecProvenance string + SpecGenerator *artifact.GeneratorStamp + SpecConcurrency []int ExistingReports []SQLiteReportExport ArtifactSummaries []SQLiteReportArtifactSummary MeasurementMetrics map[int64]map[string]SQLiteReportMetric @@ -449,6 +452,7 @@ func LoadSQLiteReport(path string) (SQLiteReportDocument, error) { loadSQLiteReportEventCounts, loadSQLiteReportNotableEvents, loadSQLiteReportCommands, + loadSQLiteReportSpecProvenance, loadSQLiteReportExports, loadSQLiteReportArtifactSummaries, } { @@ -459,7 +463,8 @@ func LoadSQLiteReport(path string) (SQLiteReportDocument, error) { doc.Measurements, doc.RepeatDetails = aggregateRepeatMeasurements(doc.Measurements) doc.Legend = ReportMetrics doc.MetadataItems = sqliteReportMetadataItems(doc) - doc.ThroughputRows = sqliteReportThroughputRows(doc) + realRows := sqliteReportThroughputRows(doc) + doc.ThroughputRows = append(realRows, trimmedThroughputRows(doc, realRows)...) doc.ThroughputGroups = sqliteReportThroughputGroups(doc.ThroughputRows) doc.PhaseSections = sqliteReportPhaseSections(doc.Measurements) doc.Charts = sqliteReportCharts(doc.Measurements) @@ -1346,8 +1351,116 @@ func sqliteReportCharts(measurements []SQLiteReportMeasurement) []SQLiteReportCh return []SQLiteReportChart{{Title: "Aggregate Output Throughput", Unit: "tok/s", Height: 24 + len(bars)*28, Bars: bars}} } +// loadSQLiteReportSpecProvenance verifies the latest run's original spec +// against its generator stamp: generated, edited after generation, or +// custom (no stamp). Verification happens here, from the stored bytes — +// the label is never taken on trust from the runner. +func loadSQLiteReportSpecProvenance(db *sql.DB, doc *SQLiteReportDocument) error { + var content string + err := db.QueryRow(`SELECT content FROM specs WHERE run_id = ? AND kind = 'original'`, doc.Run.ID).Scan(&content) + if err == sql.ErrNoRows { + doc.SpecProvenance = artifact.SpecProvenanceCustom + return nil + } + if err != nil { + return err + } + status, stamp := artifact.VerifySpecProvenance([]byte(content)) + doc.SpecProvenance = status + doc.SpecGenerator = stamp + if status == artifact.SpecProvenanceGenerated && stamp != nil && len(stamp.Intent) > 0 { + var intent struct { + Concurrency []int `json:"concurrency"` + } + if err := json.Unmarshal(stamp.Intent, &intent); err == nil { + doc.SpecConcurrency = intent.Concurrency + } + } + return nil +} + +func specProvenanceDisplay(doc SQLiteReportDocument) string { + switch doc.SpecProvenance { + case artifact.SpecProvenanceGenerated: + return "Generated default sweep (" + doc.SpecGenerator.Tool + ")" + case artifact.SpecProvenanceEdited: + return "Custom grid (edited after generation)" + default: + return "Custom grid (hand-authored)" + } +} + +// trimmedThroughputRows synthesizes rows for author-trimmed ladder points so +// declared trims render like adaptive skips, never as silent holes. Only +// verified generated specs are trusted for this. +func trimmedThroughputRows(doc SQLiteReportDocument, existing []SQLiteReportThroughputRow) []SQLiteReportThroughputRow { + if doc.SpecProvenance != artifact.SpecProvenanceGenerated || doc.SpecGenerator == nil { + return nil + } + ladder := doc.SpecConcurrency + if len(ladder) == 0 { + return nil + } + // A real measurement — from any run in a model-level artifact — always + // wins over a synthesized trim marker for the same point. + measured := map[string]struct{}{} + for _, row := range existing { + measured[fmt.Sprintf("%d/%s/%d", row.ContextTarget, row.Mode, row.Concurrency)] = struct{}{} + } + var rows []SQLiteReportThroughputRow + for _, trim := range doc.SpecGenerator.LadderTrims { + label := contextLabel(trim.Context) + for _, concurrency := range ladder { + if concurrency <= trim.MaxConcurrency { + continue + } + for _, mode := range []string{"decode", "prefill"} { + if _, ok := measured[fmt.Sprintf("%d/%s/%d", trim.Context, mode, concurrency)]; ok { + continue + } + rows = append(rows, SQLiteReportThroughputRow{ + Phase: bench.PhaseTitle(mode), + Mode: mode, + RunID: doc.Run.ID, + Profile: label, + Model: doc.Run.Name, + ContextWindow: trim.Context, + ContextLabel: "unverified (declared " + label + " active)", + ContextSortKey: trim.Context, + ContextTarget: trim.Context, + ContextSemantics: "active", + Concurrency: concurrency, + Shape: "-", + ThroughputTokS: "trimmed", + PerUserTokS: "trimmed", + TTFTMeanMS: "-", + TTFTP99MS: "-", + LatencyP95MS: "-", + Status: "skipped", + FailureLabel: "trimmed", + FailureReason: "trimmed by author: " + trim.Reason, + Detail: SQLiteReportCellDetail{ + Available: true, + Phase: bench.PhaseTitle(mode), + Mode: mode, + Status: "skipped", + FailureLabel: "trimmed", + FailureReason: "trimmed by author: " + trim.Reason, + Profile: label, + ContextLabel: "unverified (declared " + label + " active)", + ContextWindow: trim.Context, + Concurrency: concurrency, + }, + }) + } + } + } + return rows +} + func sqliteReportMetadataItems(doc SQLiteReportDocument) []SQLiteReportMetadataItem { items := []SQLiteReportMetadataItem{ + {Label: "Spec", Value: specProvenanceDisplay(doc)}, {Label: "Engine", Value: joinUnique(engineSummaries(doc.Engines), ", ")}, {Label: "Runs", Value: fmt.Sprint(len(doc.Runs))}, {Label: "Hardware", Value: bench.FirstNonEmpty(doc.Run.Hardware, "-")}, diff --git a/internal/report/html_test.go b/internal/report/html_test.go index fc4d601..ed78b3f 100644 --- a/internal/report/html_test.go +++ b/internal/report/html_test.go @@ -2,6 +2,7 @@ package report import ( "database/sql" + "encoding/json" "os" "path/filepath" "strings" @@ -1001,3 +1002,87 @@ func TestCellDetailIncludesMetrics(t *testing.T) { } } } + +func TestSpecProvenanceRendersInReport(t *testing.T) { + artifactPath := filepath.Join(t.TempDir(), "run.sqlite") + createTestSQLiteHTMLArtifact(t, artifactPath, "Provenance") + doc, err := LoadSQLiteReport(artifactPath) + if err != nil { + t.Fatal(err) + } + // The fixture's original spec has no generator stamp. + found := false + for _, item := range doc.MetadataItems { + if item.Label == "Spec" { + found = true + if item.Value != "Custom grid (hand-authored)" { + t.Fatalf("spec provenance = %q, want custom grid for an unstamped spec", item.Value) + } + } + } + if !found { + t.Fatal("metadata missing Spec provenance item") + } +} + +func TestGeneratedSpecTrimsRenderAsRows(t *testing.T) { + artifactPath := filepath.Join(t.TempDir(), "run.sqlite") + createTestSQLiteHTMLArtifact(t, artifactPath, "Trims") + base := map[string]any{"version": "1", "name": "stamped", "model": "example/model"} + stamp := artifact.GeneratorStamp{ + Tool: "localperf sweep plan", + Version: "1", + Intent: json.RawMessage(`{"concurrency":[1,4,8,16]}`), + LadderTrims: []artifact.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, + } + base["generator"] = stamp + unhashed, err := json.Marshal(base) + if err != nil { + t.Fatal(err) + } + // The hash covers the stamp's trusted fields; only content_hash itself + // is excluded. + hash, err := artifact.CanonicalSpecHash(unhashed) + if err != nil { + t.Fatal(err) + } + stamp.ContentHash = hash + base["generator"] = stamp + content, err := json.Marshal(base) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", artifactPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE specs SET content = ? WHERE kind = 'original'`, string(content)); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + doc, err := LoadSQLiteReport(artifactPath) + if err != nil { + t.Fatal(err) + } + trimmed := []SQLiteReportThroughputRow{} + for _, row := range doc.ThroughputRows { + if row.FailureLabel == "trimmed" { + trimmed = append(trimmed, row) + } + } + // c16 above the trim cap, decode + prefill. + if len(trimmed) != 2 { + t.Fatalf("trimmed rows = %d, want 2 synthesized rows for c16", len(trimmed)) + } + row := trimmed[0] + if row.Concurrency != 16 || row.ContextTarget != 65536 || !strings.Contains(row.FailureReason, "12 GiB KV budget") { + t.Fatalf("trimmed row = %+v, want c16 at 64k with the declared reason", row) + } + for _, item := range doc.MetadataItems { + if item.Label == "Spec" && !strings.Contains(item.Value, "Generated default sweep") { + t.Fatalf("spec provenance = %q, want generated", item.Value) + } + } +} diff --git a/internal/sweepplan/provenance_test.go b/internal/sweepplan/provenance_test.go new file mode 100644 index 0000000..748f4b1 --- /dev/null +++ b/internal/sweepplan/provenance_test.go @@ -0,0 +1,131 @@ +package sweepplan + +import ( + "testing" + + "github.com/dutifuldev/localperf/internal/vllmbench" +) + +func TestPlanStampsVerifiableProvenance(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{8192, 65536}, + Trims: []vllmbench.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, + }) + if err != nil { + t.Fatal(err) + } + if spec.Generator == nil || spec.Generator.Tool != "localperf sweep plan" { + t.Fatalf("generator stamp = %+v, want tool identity", spec.Generator) + } + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceGenerated { + t.Fatalf("provenance = %q, want generated for an unedited spec", got) + } + spec.Workloads[0].NumPrompts = 999 + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceEdited { + t.Fatalf("provenance = %q, want edited after mutation", got) + } + // The stamp's trusted fields are covered by the hash: reports rely on + // declared trims, so editing them must demote the spec too. + fresh, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{8192, 65536}, + Trims: []vllmbench.LadderTrim{{Context: 65536, MaxConcurrency: 8, Reason: "12 GiB KV budget"}}, + }) + if err != nil { + t.Fatal(err) + } + fresh.Generator.LadderTrims = append(fresh.Generator.LadderTrims, vllmbench.LadderTrim{Context: 8192, MaxConcurrency: 1, Reason: "invented"}) + if got := vllmbench.SpecProvenance(fresh); got != vllmbench.SpecProvenanceEdited { + t.Fatalf("provenance = %q, want edited after tampering with declared trims", got) + } + spec.Generator = nil + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceCustom { + t.Fatalf("provenance = %q, want custom without a stamp", got) + } +} + +func TestTrimsCapLaddersAndRideTheStamp(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{32768, 65536}, + Trims: []vllmbench.LadderTrim{ + {Context: 32768, MaxConcurrency: 16, Reason: "KV budget"}, + {Context: 65536, MaxConcurrency: 8, Reason: "KV budget"}, + }, + }) + if err != nil { + t.Fatal(err) + } + maxByWorkload := map[string]int{} + for _, workload := range spec.Workloads { + top := 0 + for _, value := range workload.MaxConcurrency { + if value > top { + top = value + } + } + maxByWorkload[workload.Name] = top + } + if maxByWorkload["decode-32k"] != 16 || maxByWorkload["prefill-32k"] != 16 { + t.Fatalf("32k ladders = %v, want capped at 16", maxByWorkload) + } + if maxByWorkload["decode-64k"] != 8 || maxByWorkload["prefill-64k"] != 8 { + t.Fatalf("64k ladders = %v, want capped at 8", maxByWorkload) + } + if len(spec.Generator.LadderTrims) != 2 { + t.Fatalf("stamped trims = %+v, want both declared trims", spec.Generator.LadderTrims) + } + for _, profile := range spec.Profiles { + if profile.Name == "64k" && profile.MaxNumSeqs != 8 { + t.Fatalf("64k max_num_seqs = %d, want sized to trimmed ladder", profile.MaxNumSeqs) + } + } +} + +func TestRuntimeIntentReachesSpec(t *testing.T) { + spec, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{8192}, + VLLMCommand: "/opt/runtimes/vllm/bin/vllm", + GPUMemoryUtilization: 0.4, + KVCacheMemoryBytes: 12 << 30, + }) + if err != nil { + t.Fatal(err) + } + if spec.Runner.VLLMCommand != "/opt/runtimes/vllm/bin/vllm" { + t.Fatalf("vllm command = %q, want intent runtime path", spec.Runner.VLLMCommand) + } + if spec.Runner.VLLMBenchCommand != "/opt/runtimes/vllm/bin/vllm" { + t.Fatalf("vllm bench command = %q, want the same runtime as serve", spec.Runner.VLLMBenchCommand) + } + for _, profile := range spec.Profiles { + if profile.GPUMemoryUtilization != 0.4 { + t.Fatalf("profile %s gpu mem = %v, want 0.4", profile.Name, profile.GPUMemoryUtilization) + } + found := false + for index, arg := range profile.Args { + if arg == "--kv-cache-memory-bytes" && index+1 < len(profile.Args) && profile.Args[index+1] == "12884901888" { + found = true + } + } + if !found { + t.Fatalf("profile %s args = %v, want kv cache bytes pinned", profile.Name, profile.Args) + } + } + if got := vllmbench.SpecProvenance(spec); got != vllmbench.SpecProvenanceGenerated { + t.Fatalf("provenance = %q, want generated", got) + } +} + +func TestTrimWithoutReasonIsRejected(t *testing.T) { + _, err := Plan(PlanRequest{ + Model: "example/model", + Contexts: []int{65536}, + Trims: []vllmbench.LadderTrim{{Context: 65536, MaxConcurrency: 8}}, + }) + if err == nil { + t.Fatal("expected a validation error for a trim without a reason") + } +} diff --git a/internal/sweepplan/sweepplan.go b/internal/sweepplan/sweepplan.go index adef17a..d48d2fb 100644 --- a/internal/sweepplan/sweepplan.go +++ b/internal/sweepplan/sweepplan.go @@ -8,7 +8,9 @@ package sweepplan import ( + "encoding/json" "fmt" + "strconv" "strings" "github.com/dutifuldev/localperf/internal/vllmbench" @@ -16,29 +18,41 @@ import ( type PlanRequest struct { // Model is the model identifier to benchmark (required). - Model string + Model string `json:"model"` // Contexts is the active-context ladder in tokens, e.g. 8192..131072. - Contexts []int + Contexts []int `json:"contexts,omitempty"` // Concurrency levels per point, e.g. 1,4,8,16,32. - Concurrency []int + Concurrency []int `json:"concurrency,omitempty"` // Repeats per measurement; defaults to 1. - Repeats int + Repeats int `json:"repeats,omitempty"` // NumPrompts fixes the request count per measurement; when 0, prompts // scale with concurrency via PromptsPerUser instead. - NumPrompts int + NumPrompts int `json:"num_prompts,omitempty"` // PromptsPerUser scales requests with concurrency // (num_prompts = max(8, PromptsPerUser * concurrency)); defaults to 2. - PromptsPerUser int + PromptsPerUser int `json:"prompts_per_user,omitempty"` // IncludeReference adds the 4k max-throughput-reference capacity family. - IncludeReference bool + IncludeReference bool `json:"include_reference,omitempty"` // IncludeStress adds long-output decode spot checks (4096 tokens at // 32k c4 and 64k c1/c4 when those contexts are in the ladder) and the // 128k points at c1/c4. - IncludeStress bool + IncludeStress bool `json:"include_stress,omitempty"` // MinMemAvailableGiB is the safety memory floor; defaults to 40. - MinMemAvailableGiB float64 + MinMemAvailableGiB float64 `json:"min_mem_available_gib,omitempty"` // BasePort is the first server port; defaults to 8100. - BasePort int + BasePort int `json:"base_port,omitempty"` + // VLLMCommand overrides the vllm executable for managed serves — the + // machine-specific runtime path that previously forced hand-editing. + VLLMCommand string `json:"vllm_command,omitempty"` + // GPUMemoryUtilization applies to every generated profile when set. + GPUMemoryUtilization float64 `json:"gpu_memory_utilization,omitempty"` + // KVCacheMemoryBytes pins vLLM's KV cache size on every profile via + // --kv-cache-memory-bytes when set. + KVCacheMemoryBytes int64 `json:"kv_cache_memory_bytes,omitempty"` + // Trims are declared author decisions to cap a context's concurrency + // ladder; each needs a reason and renders in reports like an adaptive + // skip, never as a silent hole. + Trims []vllmbench.LadderTrim `json:"trims,omitempty"` } const ( @@ -102,6 +116,17 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { if len(request.Contexts) == 0 && !request.IncludeReference { return vllmbench.Spec{}, fmt.Errorf("at least one context point or the reference family is required") } + for index, trim := range request.Trims { + if trim.Context <= 0 || trim.MaxConcurrency <= 0 { + return vllmbench.Spec{}, fmt.Errorf("trims[%d]: context and max_concurrency must be positive", index) + } + if strings.TrimSpace(trim.Reason) == "" { + return vllmbench.Spec{}, fmt.Errorf("trims[%d]: a reason is required — declared trims render in reports", index) + } + if !containsInt(request.Contexts, trim.Context) && !(request.IncludeStress && trim.Context == stressContext) { + return vllmbench.Spec{}, fmt.Errorf("trims[%d]: context %d is not in the generated ladder", index, trim.Context) + } + } if len(request.Concurrency) == 0 { request.Concurrency = []int{1, 4, 8, 16, 32} } @@ -173,15 +198,71 @@ func Plan(request PlanRequest) (vllmbench.Spec, error) { spec.Workloads = append(spec.Workloads, stressWorkloads(contexts, request)...) } + applyRuntimeIntent(&spec, request) + // Emit the normalized spec: explicit defaults are stable to diff and // leave nothing for a reader to guess. vllmbench.ApplyDefaults(&spec) if err := vllmbench.ValidateSpec(spec); err != nil { return vllmbench.Spec{}, fmt.Errorf("generated spec failed validation (generator/validator drift): %w", err) } + if err := stampSpec(&spec, request); err != nil { + return vllmbench.Spec{}, err + } return spec, nil } +// applyRuntimeIntent carries machine-specific runtime choices into the spec +// so nothing forces hand-editing the generated file. +func applyRuntimeIntent(spec *vllmbench.Spec, request PlanRequest) { + if strings.TrimSpace(request.VLLMCommand) != "" { + spec.Runner.VLLMCommand = request.VLLMCommand + // The bench loader must run from the same runtime, or the sweep + // would serve with the requested vLLM and benchmark with whatever + // is on PATH. + spec.Runner.VLLMBenchCommand = request.VLLMCommand + } + for index := range spec.Profiles { + if request.GPUMemoryUtilization > 0 { + spec.Profiles[index].GPUMemoryUtilization = request.GPUMemoryUtilization + } + if request.KVCacheMemoryBytes > 0 { + spec.Profiles[index].Args = append(spec.Profiles[index].Args, + "--kv-cache-memory-bytes", strconv.FormatInt(request.KVCacheMemoryBytes, 10)) + } + } +} + +// generatorTool and generatorVersion identify the stamp format, not the +// binary build: the content hash is what verification relies on. +const ( + generatorTool = "localperf sweep plan" + generatorVersion = "1" +) + +// stampSpec writes the provenance record: the intent that produced the +// spec, declared ladder trims, and a content hash over the spec with the +// stamp removed. `bench run` and reports verify the hash; any edit after +// generation demotes the spec to a custom grid. +func stampSpec(spec *vllmbench.Spec, request PlanRequest) error { + intent, err := json.Marshal(request) + if err != nil { + return err + } + spec.Generator = &vllmbench.GeneratorStamp{ + Tool: generatorTool, + Version: generatorVersion, + Intent: intent, + LadderTrims: append([]vllmbench.LadderTrim{}, request.Trims...), + } + hash, err := vllmbench.SpecContentHash(*spec) + if err != nil { + return err + } + spec.Generator.ContentHash = hash + return nil +} + func sweepProfile(name string, maxModelLen, port, maxNumSeqs int) vllmbench.Profile { return vllmbench.Profile{ Name: name, @@ -226,7 +307,7 @@ func stressWorkloads(contexts []int, request PlanRequest) []vllmbench.Workload { // ladder plus any stress spot checks attached to that context; a stress c4 // point must not run against a server that only admits c1. func contextMaxSeqs(context int, request PlanRequest) int { - seqs := maxConcurrencyOf(pointConcurrency(context, request.Concurrency)) + seqs := maxConcurrencyOf(pointConcurrency(context, request.Concurrency, request.Trims)) if request.IncludeStress { for _, spot := range stressSpots { if spot.context == context { @@ -258,13 +339,22 @@ func containsInt(values []int, wanted int) bool { // pointConcurrency caps contexts >= 128k at c4; higher concurrency at those // KV budgets is stress territory, not the default grid. -func pointConcurrency(context int, concurrency []int) []int { - if context < stressContext { +func pointConcurrency(context int, concurrency []int, trims []vllmbench.LadderTrim) []int { + cap := 0 + if context >= stressContext { + cap = highContextConcurrencyCap + } + for _, trim := range trims { + if trim.Context == context && (cap == 0 || trim.MaxConcurrency < cap) { + cap = trim.MaxConcurrency + } + } + if cap == 0 { return append([]int{}, concurrency...) } capped := []int{} for _, value := range concurrency { - if value <= highContextConcurrencyCap { + if value <= cap { capped = append(capped, value) } } @@ -294,7 +384,7 @@ func sweepWorkload(name, phase, profile string, target int, semantics string, in }, NumPrompts: request.NumPrompts, PromptsPerUser: request.PromptsPerUser, - MaxConcurrency: pointConcurrency(target, request.Concurrency), + MaxConcurrency: pointConcurrency(target, request.Concurrency, request.Trims), IgnoreEOS: true, Temperature: &temperature, } diff --git a/internal/sweepplan/testdata/default-sweep-8k.golden.json b/internal/sweepplan/testdata/default-sweep-8k.golden.json index 9dd3b8d..743ec98 100644 --- a/internal/sweepplan/testdata/default-sweep-8k.golden.json +++ b/internal/sweepplan/testdata/default-sweep-8k.golden.json @@ -168,5 +168,24 @@ "ignore_eos": true, "temperature": 0 } - ] + ], + "generator": { + "tool": "localperf sweep plan", + "version": "1", + "intent": { + "model": "example/model", + "contexts": [ + 8192 + ], + "concurrency": [ + 1, + 4 + ], + "prompts_per_user": 2, + "include_reference": true, + "min_mem_available_gib": 40, + "base_port": 8100 + }, + "content_hash": "4f4964d8f7673921cc1f398579f70cf856e0adca8487cb2516926ea6ef410cac" + } } diff --git a/internal/vllmbench/config.go b/internal/vllmbench/config.go index 1d12bdb..8b21e15 100644 --- a/internal/vllmbench/config.go +++ b/internal/vllmbench/config.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "github.com/dutifuldev/localperf/internal/artifact" + "github.com/dutifuldev/localperf/internal/bench" "github.com/dutifuldev/localperf/internal/collections" ) @@ -40,18 +42,58 @@ const ( ) type Spec struct { - Version string `json:"version"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - Model string `json:"model"` - OutputDir string `json:"output_dir,omitempty"` - Env map[string]string `json:"env,omitempty"` - Engines []EngineConfig `json:"engines,omitempty"` - Runner RunnerConfig `json:"runner"` - Safety SafetyConfig `json:"safety"` - Warmup WarmupConfig `json:"warmup,omitempty"` - Profiles []Profile `json:"profiles"` - Workloads []Workload `json:"workloads"` + Version string `json:"version"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Model string `json:"model"` + OutputDir string `json:"output_dir,omitempty"` + Env map[string]string `json:"env,omitempty"` + Engines []EngineConfig `json:"engines,omitempty"` + Runner RunnerConfig `json:"runner"` + Safety SafetyConfig `json:"safety"` + Warmup WarmupConfig `json:"warmup,omitempty"` + Profiles []Profile `json:"profiles"` + Workloads []Workload `json:"workloads"` + Generator *artifact.GeneratorStamp `json:"generator,omitempty"` + + // Provenance is computed at load time from the generator stamp; it is + // never serialized or taken on trust. + Provenance string `json:"-"` +} + +// SpecContentHash hashes the canonical spec document with the generator +// stamp removed; see artifact.CanonicalSpecHash. +func SpecContentHash(spec Spec) (string, error) { + data, err := json.Marshal(spec) + if err != nil { + return "", err + } + return artifact.CanonicalSpecHash(data) +} + +// Provenance types and statuses re-exported for callers of this package, +// so higher layers do not need to import the artifact format directly. +type ( + GeneratorStamp = artifact.GeneratorStamp + LadderTrim = artifact.LadderTrim +) + +const ( + SpecProvenanceGenerated = artifact.SpecProvenanceGenerated + SpecProvenanceEdited = artifact.SpecProvenanceEdited + SpecProvenanceCustom = artifact.SpecProvenanceCustom +) + +// SpecProvenance verifies a spec's generator stamp: "generated" when the +// content hash matches, "edited" when a stamp exists but the content +// changed after generation, "custom" when there is no stamp. +func SpecProvenance(spec Spec) string { + data, err := json.Marshal(spec) + if err != nil { + return artifact.SpecProvenanceCustom + } + status, _ := artifact.VerifySpecProvenance(data) + return status } type EngineConfig struct { @@ -238,6 +280,10 @@ func LoadSpec(path string) (Spec, error) { if err := json.Unmarshal(data, &spec); err != nil { return Spec{}, err } + // Verify provenance against the raw file bytes, before unmarshalling + // drops unknown fields and defaults mutate the spec: the runner's label + // must match what reports later verify from the stored bytes. + spec.Provenance, _ = artifact.VerifySpecProvenance(data) ApplyDefaults(&spec) if err := ValidateSpec(spec); err != nil { return Spec{}, err @@ -681,12 +727,40 @@ func ValidateSpec(spec Spec) error { issues = append(issues, validateEndpointBaseURLProfileUsage(spec)...) issues = append(issues, validateWarmup(spec.Warmup)...) issues = append(issues, validateWorkloads(spec.Workloads, profileNames, spec.Profiles)...) + issues = append(issues, validateGeneratorStamp(spec.Generator)...) if len(issues) > 0 { return errors.New(strings.Join(issues, "\n")) } return nil } +// validateGeneratorStamp requires every declared ladder trim to carry its +// reason: an unexplained trim is a silent hole waiting to render. +func validateGeneratorStamp(stamp *artifact.GeneratorStamp) []string { + if stamp == nil { + return nil + } + var issues []string + for index, trim := range stamp.LadderTrims { + issues = append(issues, validateLadderTrim(index, trim)...) + } + return issues +} + +func validateLadderTrim(index int, trim artifact.LadderTrim) []string { + var issues []string + if trim.Context <= 0 { + issues = append(issues, fmt.Sprintf("generator.ladder_trims[%d]: context must be positive", index)) + } + if trim.MaxConcurrency <= 0 { + issues = append(issues, fmt.Sprintf("generator.ladder_trims[%d]: max_concurrency must be positive", index)) + } + if strings.TrimSpace(trim.Reason) == "" { + issues = append(issues, fmt.Sprintf("generator.ladder_trims[%d]: reason is required", index)) + } + return issues +} + func validateSpecBasics(spec Spec) []string { var issues []string if strings.TrimSpace(spec.Name) == "" { diff --git a/internal/vllmbench/config_test.go b/internal/vllmbench/config_test.go index 5b853cd..7e7027b 100644 --- a/internal/vllmbench/config_test.go +++ b/internal/vllmbench/config_test.go @@ -3671,3 +3671,19 @@ func TestPrefixCachingResolvedFromArgs(t *testing.T) { t.Fatalf("prefix caching = %v, want unknown without any signal", got) } } + +func TestValidateLadderTrimBranches(t *testing.T) { + if issues := validateGeneratorStamp(nil); len(issues) != 0 { + t.Fatalf("nil stamp issues = %v, want none", issues) + } + stamp := &GeneratorStamp{LadderTrims: []LadderTrim{ + {Context: 0, MaxConcurrency: 8, Reason: "r"}, + {Context: 65536, MaxConcurrency: 0, Reason: "r"}, + {Context: 65536, MaxConcurrency: 8, Reason: " "}, + {Context: 65536, MaxConcurrency: 8, Reason: "ok"}, + }} + issues := validateGeneratorStamp(stamp) + if len(issues) != 3 { + t.Fatalf("issues = %v, want one per invalid trim", issues) + } +}