Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,18 @@ caseforge lint --spec openapi.yaml
--exclude-tag string Comma-separated OpenAPI tags to exclude (e.g. 'deprecated')
--auth-bootstrap Wrap all secured-endpoint cases with an auth setup step
--with-oracles Mine response body constraints via LLM and inject as assertions (requires LLM)
--force Regenerate even when spec hash matches existing output
--annotation-batch N Number of operations to annotate per LLM call (0 = one call per op; recommended: 8–20)
```

**Smart regeneration:** `gen` hashes the spec file on each run. If the hash matches the previously generated output, it exits early with a `✓ Spec unchanged` message. Use `--force` to bypass this.

**Batch annotation:** By default each operation is annotated in a separate LLM call. On large specs (20+ operations) this can take several minutes. `--annotation-batch 10` groups 10 operations per call, reducing round-trips dramatically.

**Rate-limit backoff:** 429 responses from LLM providers trigger automatic exponential backoff (5 s → 15 s → 30 s → 60 s) before retrying.

**Hurl output headers:** Every generated `.hurl` file includes `# case_id=` and `# case_name=` comment headers for traceability.

### `caseforge run`

```
Expand Down
2 changes: 1 addition & 1 deletion cmd/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ func runGen(cmd *cobra.Command, args []string) error {
if genMaxCasesPerOp > 0 {
engine.SetMaxCasesPerOp(genMaxCasesPerOp)
}
if genAnnotationBatch > 0 {
if genAnnotationBatch >= 1 {
engine.SetAnnotationBatch(genAnnotationBatch)
}
newCases, err := engine.Generate(parsedSpec)
Expand Down
13 changes: 12 additions & 1 deletion internal/mcp/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,18 @@ func TestNewServerReturnsNonNil(t *testing.T) {
}

func TestServerHasGenerateTestCasesTool(t *testing.T) {
assert.NotNil(t, generateTestCasesTool())
tool := generateTestCasesTool()
require.NotNil(t, tool)
assert.Equal(t, "generate_test_cases", tool.Name)

rawSchema, _ := tool.InputSchema.(json.RawMessage)
var schema map[string]any
require.NoError(t, json.Unmarshal(rawSchema, &schema))
required, _ := schema["required"].([]any)
assert.Contains(t, required, "spec")

props, _ := schema["properties"].(map[string]any)
assert.Contains(t, props, "annotation_batch", "annotation_batch must be in schema for MCP clients to discover it")
}

func TestServerHasLintSpecTool(t *testing.T) {
Expand Down
14 changes: 11 additions & 3 deletions internal/mcp/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ func generateTestCasesTool() *mcpsdk.Tool {
"type": "string",
"description": "Output format: hurl|markdown|csv|postman|k6 (default: hurl)",
"enum": ["hurl", "markdown", "csv", "postman", "k6"]
},
"annotation_batch": {
"type": "integer",
"description": "Number of operations to annotate per LLM call; reduces round-trips on large specs (0 = one call per op, recommended: 8–20)"
}
},
"required": ["spec"]
Expand All @@ -48,9 +52,10 @@ func generateTestCasesTool() *mcpsdk.Tool {

func makeGenerateHandler(ctx context.Context, req *mcpsdk.CallToolRequest) (*mcpsdk.CallToolResult, error) {
var args struct {
Spec string `json:"spec"`
Output string `json:"output"`
Format string `json:"format"`
Spec string `json:"spec"`
Output string `json:"output"`
Format string `json:"format"`
AnnotationBatch int `json:"annotation_batch"`
}
if err := json.Unmarshal(req.Params.Arguments, &args); err != nil {
r := &mcpsdk.CallToolResult{}
Expand Down Expand Up @@ -88,6 +93,9 @@ func makeGenerateHandler(ctx context.Context, req *mcpsdk.CallToolRequest) (*mcp
)
engine.AddSpecTechnique(methodology.NewChainTechnique())
engine.AddSpecTechnique(methodology.NewSecuritySpecTechnique())
if args.AnnotationBatch >= 1 {
engine.SetAnnotationBatch(args.AnnotationBatch)
}

cases, err := engine.Generate(parsedSpec)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/methodology/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ func (e *Engine) SetMaxCasesPerOp(n int) {

// SetAnnotationBatch sets the number of operations to annotate per LLM call.
// 0 (default) uses sequential mode: one call per operation.
// Values > 0 batch that many operations into a single call, reducing round-trips
// at the cost of larger prompts. Recommended range: 5–20.
// Values >= 1 batch that many operations into a single call, reducing round-trips
// at the cost of larger prompts. Recommended range: 8–20.
func (e *Engine) SetAnnotationBatch(n int) {
e.annotationBatch = n
}
Expand Down
Loading