From b8ba02f9df479340e89a5fc8d44c3b3030126b48 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Fri, 8 May 2026 08:01:52 +0800 Subject: [PATCH 1/2] docs(gen): document --force, --annotation-batch, dedup, and hurl headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README: - Add --force and --annotation-batch N flags to gen command reference - Add Smart regeneration explanation (spec-hash dedup + --force) - Add Batch annotation explanation (--annotation-batch performance tip) - Add Rate-limit backoff note (5s→15s→30s→60s exponential backoff) - Add Hurl output headers note (# case_id= and # case_name= fields) MCP generate_test_cases tool: - Add force (boolean) parameter to InputSchema - Add annotation_batch (integer) parameter to InputSchema - Wire annotation_batch into engine.SetAnnotationBatch() in the handler --- README.md | 10 ++++++++++ internal/mcp/tools.go | 19 ++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 48b90f1..90aa740 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,18 @@ caseforge lint --spec openapi.yaml --exclude-tag string Comma-separated OpenAPI tags to exclude (e.g. 'deprecated') --auth-bootstrap Wrap all secured-endpoint cases with an auth setup step --with-oracles Mine response body constraints via LLM and inject as assertions (requires LLM) +--force Regenerate even when spec hash matches existing output +--annotation-batch N Number of operations to annotate per LLM call (0 = one call per op; recommended: 8–20) ``` +**Smart regeneration:** `gen` hashes the spec file on each run. If the hash matches the previously generated output, it exits early with a `✓ Spec unchanged` message. Use `--force` to bypass this. + +**Batch annotation:** By default each operation is annotated in a separate LLM call. On large specs (20+ operations) this can take several minutes. `--annotation-batch 10` groups 10 operations per call, reducing round-trips dramatically. + +**Rate-limit backoff:** 429 responses from LLM providers trigger automatic exponential backoff (5 s → 15 s → 30 s → 60 s) before retrying. + +**Hurl output headers:** Every generated `.hurl` file includes `# case_id=` and `# case_name=` comment headers for traceability. + ### `caseforge run` ``` diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index 34afd88..28cde45 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -39,6 +39,14 @@ func generateTestCasesTool() *mcpsdk.Tool { "type": "string", "description": "Output format: hurl|markdown|csv|postman|k6 (default: hurl)", "enum": ["hurl", "markdown", "csv", "postman", "k6"] + }, + "force": { + "type": "boolean", + "description": "Regenerate even when spec hash matches existing output (default: false)" + }, + "annotation_batch": { + "type": "integer", + "description": "Number of operations to annotate per LLM call; reduces round-trips on large specs (0 = one call per op, recommended: 8–20)" } }, "required": ["spec"] @@ -48,9 +56,11 @@ func generateTestCasesTool() *mcpsdk.Tool { func makeGenerateHandler(ctx context.Context, req *mcpsdk.CallToolRequest) (*mcpsdk.CallToolResult, error) { var args struct { - Spec string `json:"spec"` - Output string `json:"output"` - Format string `json:"format"` + Spec string `json:"spec"` + Output string `json:"output"` + Format string `json:"format"` + Force bool `json:"force"` + AnnotationBatch int `json:"annotation_batch"` } if err := json.Unmarshal(req.Params.Arguments, &args); err != nil { r := &mcpsdk.CallToolResult{} @@ -88,6 +98,9 @@ func makeGenerateHandler(ctx context.Context, req *mcpsdk.CallToolRequest) (*mcp ) engine.AddSpecTechnique(methodology.NewChainTechnique()) engine.AddSpecTechnique(methodology.NewSecuritySpecTechnique()) + if args.AnnotationBatch >= 1 { + engine.SetAnnotationBatch(args.AnnotationBatch) + } cases, err := engine.Generate(parsedSpec) if err != nil { From 49cb3b35d5ab845c1b1f2a36389771beecc18c15 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Fri, 8 May 2026 08:55:55 +0800 Subject: [PATCH 2/2] fix(mcp): address docs review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove force from generate_test_cases MCP schema — the MCP handler has no spec-hash dedup logic (it always regenerates), so the parameter was a silent no-op that misled callers - Align SetAnnotationBatch docstring to "8–20" (was "5–20") to match README - Standardize annotation_batch guard to >= 1 in cmd/gen.go (was > 0) - Expand TestServerHasGenerateTestCasesTool to assert annotation_batch is present in the tool's InputSchema properties --- cmd/gen.go | 2 +- internal/mcp/server_test.go | 13 ++++++++++++- internal/mcp/tools.go | 5 ----- internal/methodology/engine.go | 4 ++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/cmd/gen.go b/cmd/gen.go index ad06dc1..c297e64 100644 --- a/cmd/gen.go +++ b/cmd/gen.go @@ -403,7 +403,7 @@ func runGen(cmd *cobra.Command, args []string) error { if genMaxCasesPerOp > 0 { engine.SetMaxCasesPerOp(genMaxCasesPerOp) } - if genAnnotationBatch > 0 { + if genAnnotationBatch >= 1 { engine.SetAnnotationBatch(genAnnotationBatch) } newCases, err := engine.Generate(parsedSpec) diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go index b43b15a..677c163 100644 --- a/internal/mcp/server_test.go +++ b/internal/mcp/server_test.go @@ -27,7 +27,18 @@ func TestNewServerReturnsNonNil(t *testing.T) { } func TestServerHasGenerateTestCasesTool(t *testing.T) { - assert.NotNil(t, generateTestCasesTool()) + tool := generateTestCasesTool() + require.NotNil(t, tool) + assert.Equal(t, "generate_test_cases", tool.Name) + + rawSchema, _ := tool.InputSchema.(json.RawMessage) + var schema map[string]any + require.NoError(t, json.Unmarshal(rawSchema, &schema)) + required, _ := schema["required"].([]any) + assert.Contains(t, required, "spec") + + props, _ := schema["properties"].(map[string]any) + assert.Contains(t, props, "annotation_batch", "annotation_batch must be in schema for MCP clients to discover it") } func TestServerHasLintSpecTool(t *testing.T) { diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index 28cde45..54844a6 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -40,10 +40,6 @@ func generateTestCasesTool() *mcpsdk.Tool { "description": "Output format: hurl|markdown|csv|postman|k6 (default: hurl)", "enum": ["hurl", "markdown", "csv", "postman", "k6"] }, - "force": { - "type": "boolean", - "description": "Regenerate even when spec hash matches existing output (default: false)" - }, "annotation_batch": { "type": "integer", "description": "Number of operations to annotate per LLM call; reduces round-trips on large specs (0 = one call per op, recommended: 8–20)" @@ -59,7 +55,6 @@ func makeGenerateHandler(ctx context.Context, req *mcpsdk.CallToolRequest) (*mcp Spec string `json:"spec"` Output string `json:"output"` Format string `json:"format"` - Force bool `json:"force"` AnnotationBatch int `json:"annotation_batch"` } if err := json.Unmarshal(req.Params.Arguments, &args); err != nil { diff --git a/internal/methodology/engine.go b/internal/methodology/engine.go index 2bf976d..0219ee6 100644 --- a/internal/methodology/engine.go +++ b/internal/methodology/engine.go @@ -87,8 +87,8 @@ func (e *Engine) SetMaxCasesPerOp(n int) { // SetAnnotationBatch sets the number of operations to annotate per LLM call. // 0 (default) uses sequential mode: one call per operation. -// Values > 0 batch that many operations into a single call, reducing round-trips -// at the cost of larger prompts. Recommended range: 5–20. +// Values >= 1 batch that many operations into a single call, reducing round-trips +// at the cost of larger prompts. Recommended range: 8–20. func (e *Engine) SetAnnotationBatch(n int) { e.annotationBatch = n }