diff --git a/README.md b/README.md index 48b90f1..90aa740 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,18 @@ caseforge lint --spec openapi.yaml --exclude-tag string Comma-separated OpenAPI tags to exclude (e.g. 'deprecated') --auth-bootstrap Wrap all secured-endpoint cases with an auth setup step --with-oracles Mine response body constraints via LLM and inject as assertions (requires LLM) +--force Regenerate even when spec hash matches existing output +--annotation-batch N Number of operations to annotate per LLM call (0 = one call per op; recommended: 8–20) ``` +**Smart regeneration:** `gen` hashes the spec file on each run. If the hash matches the previously generated output, it exits early with a `✓ Spec unchanged` message. Use `--force` to bypass this. + +**Batch annotation:** By default each operation is annotated in a separate LLM call. On large specs (20+ operations) this can take several minutes. `--annotation-batch 10` groups 10 operations per call, reducing round-trips dramatically. + +**Rate-limit backoff:** 429 responses from LLM providers trigger automatic exponential backoff (5 s → 15 s → 30 s → 60 s) before retrying. + +**Hurl output headers:** Every generated `.hurl` file includes `# case_id=` and `# case_name=` comment headers for traceability. + ### `caseforge run` ``` diff --git a/cmd/gen.go b/cmd/gen.go index ad06dc1..c297e64 100644 --- a/cmd/gen.go +++ b/cmd/gen.go @@ -403,7 +403,7 @@ func runGen(cmd *cobra.Command, args []string) error { if genMaxCasesPerOp > 0 { engine.SetMaxCasesPerOp(genMaxCasesPerOp) } - if genAnnotationBatch > 0 { + if genAnnotationBatch >= 1 { engine.SetAnnotationBatch(genAnnotationBatch) } newCases, err := engine.Generate(parsedSpec) diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go index b43b15a..677c163 100644 --- a/internal/mcp/server_test.go +++ b/internal/mcp/server_test.go @@ -27,7 +27,18 @@ func TestNewServerReturnsNonNil(t *testing.T) { } func TestServerHasGenerateTestCasesTool(t *testing.T) { - assert.NotNil(t, generateTestCasesTool()) + tool := generateTestCasesTool() + require.NotNil(t, tool) + assert.Equal(t, "generate_test_cases", tool.Name) + + rawSchema, _ := tool.InputSchema.(json.RawMessage) + var schema map[string]any + require.NoError(t, json.Unmarshal(rawSchema, &schema)) + required, _ := schema["required"].([]any) + assert.Contains(t, required, "spec") + + props, _ := schema["properties"].(map[string]any) + assert.Contains(t, props, "annotation_batch", "annotation_batch must be in schema for MCP clients to discover it") } func TestServerHasLintSpecTool(t *testing.T) { diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index 34afd88..54844a6 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -39,6 +39,10 @@ func generateTestCasesTool() *mcpsdk.Tool { "type": "string", "description": "Output format: hurl|markdown|csv|postman|k6 (default: hurl)", "enum": ["hurl", "markdown", "csv", "postman", "k6"] + }, + "annotation_batch": { + "type": "integer", + "description": "Number of operations to annotate per LLM call; reduces round-trips on large specs (0 = one call per op, recommended: 8–20)" } }, "required": ["spec"] @@ -48,9 +52,10 @@ func generateTestCasesTool() *mcpsdk.Tool { func makeGenerateHandler(ctx context.Context, req *mcpsdk.CallToolRequest) (*mcpsdk.CallToolResult, error) { var args struct { - Spec string `json:"spec"` - Output string `json:"output"` - Format string `json:"format"` + Spec string `json:"spec"` + Output string `json:"output"` + Format string `json:"format"` + AnnotationBatch int `json:"annotation_batch"` } if err := json.Unmarshal(req.Params.Arguments, &args); err != nil { r := &mcpsdk.CallToolResult{} @@ -88,6 +93,9 @@ func makeGenerateHandler(ctx context.Context, req *mcpsdk.CallToolRequest) (*mcp ) engine.AddSpecTechnique(methodology.NewChainTechnique()) engine.AddSpecTechnique(methodology.NewSecuritySpecTechnique()) + if args.AnnotationBatch >= 1 { + engine.SetAnnotationBatch(args.AnnotationBatch) + } cases, err := engine.Generate(parsedSpec) if err != nil { diff --git a/internal/methodology/engine.go b/internal/methodology/engine.go index 2bf976d..0219ee6 100644 --- a/internal/methodology/engine.go +++ b/internal/methodology/engine.go @@ -87,8 +87,8 @@ func (e *Engine) SetMaxCasesPerOp(n int) { // SetAnnotationBatch sets the number of operations to annotate per LLM call. // 0 (default) uses sequential mode: one call per operation. -// Values > 0 batch that many operations into a single call, reducing round-trips -// at the cost of larger prompts. Recommended range: 5–20. +// Values >= 1 batch that many operations into a single call, reducing round-trips +// at the cost of larger prompts. Recommended range: 8–20. func (e *Engine) SetAnnotationBatch(n int) { e.annotationBatch = n }