diff --git a/README.md b/README.md index 90aa740..f6a17e1 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ caseforge lint --spec openapi.yaml | Command | Description | |---------|-------------| +| `sandbox` | Start a local HTTP mock server that generates realistic responses from an OpenAPI spec | | `chain` | Generate multi-step chain cases via BFS over the dependency graph | | `watch` | Watch a spec file and regenerate cases on change | | `suite create` | Create a `suite.json` orchestration file | @@ -157,6 +158,7 @@ caseforge lint --spec openapi.yaml --exclude-tag string Comma-separated OpenAPI tags to exclude (e.g. 'deprecated') --auth-bootstrap Wrap all secured-endpoint cases with an auth setup step --with-oracles Mine response body constraints via LLM and inject as assertions (requires LLM) +--with-sandbox Start a local sandbox server, run generated cases against it, exit non-zero on failure --force Regenerate even when spec hash matches existing output --annotation-batch N Number of operations to annotate per LLM call (0 = one call per op; recommended: 8–20) ``` @@ -335,6 +337,36 @@ via `--data-pool` to seed realistic field values into generated chain probes. --format string terminal | json (default: terminal) ``` +### `caseforge sandbox` + +Start a local HTTP mock server that serves realistic responses generated from an OpenAPI spec. Stop with Ctrl-C (SIGINT/SIGTERM triggers graceful shutdown). + +- **Use `sandbox`** for interactive development and debugging — explore with curl or Postman, inspect logs, iterate on your spec. +- **Use `gen --with-sandbox`** for CI one-shot validation — generates cases and runs them against the sandbox automatically. + +``` +--spec string OpenAPI spec file (required) +--port int Listen port; 0 = random (default: 0) +--host string Listen address (default: 127.0.0.1) +--log-level string info | warn | error | silent (default: info) +--log-file string Append JSON structured logs to file (optional) +--format string Response generation strategy: auto | schema | faker (default: auto) +``` + +`--format auto` tries strategies in order: first uses spec examples (`x-examples` / `components/examples`), then derives zero-values from the JSON Schema, then falls back to faker-generated values. + +On startup prints: `caseforge sandbox listening on http://127.0.0.1:` + +**Stateful CRUD:** `POST /resource` generates a response body, stores it, and returns the resource ID in both the body and the `X-Sandbox-ID` response header. A subsequent `GET /resource/{id}` returns the same stored object (200) or 404 if absent; `DELETE /resource/{id}` removes it and returns 204; `GET /resource` (no ID) returns all stored objects as a JSON array. + +**CI one-shot tip:** The sandbox always returns success responses, so test cases that assert 4xx status codes will fail. Pair `--with-sandbox` with `--technique equivalence_partitioning` and a spec that defines only success responses to generate only happy-path cases: + +```bash +caseforge gen --spec api.yaml --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output ./cases --with-sandbox +``` + ### `caseforge watch` ``` diff --git a/cmd/conformance.go b/cmd/conformance.go index c447af6..7be811a 100644 --- a/cmd/conformance.go +++ b/cmd/conformance.go @@ -127,7 +127,7 @@ func runConformance(cmd *cobra.Command, _ []string) error { } r := runner.NewHurlRunner() - result, runErr := r.Run(tmpDir, map[string]string{"BASE_URL": target}) + result, runErr := r.Run(tmpDir, map[string]string{"base_url": target}) if runErr != nil { return fmt.Errorf("running conformance tests: %w", runErr) } diff --git a/cmd/gen.go b/cmd/gen.go index c297e64..09788ac 100644 --- a/cmd/gen.go +++ b/cmd/gen.go @@ -9,8 +9,10 @@ import ( "path/filepath" "strings" "sync" + "time" tea "github.com/charmbracelet/bubbletea" + "github.com/fatih/color" "github.com/mattn/go-isatty" "github.com/spf13/cobra" "github.com/testmind-hq/caseforge/internal/checkpoint" @@ -22,6 +24,8 @@ import ( "github.com/testmind-hq/caseforge/internal/output/render" "github.com/testmind-hq/caseforge/internal/output/schema" "github.com/testmind-hq/caseforge/internal/output/writer" + "github.com/testmind-hq/caseforge/internal/runner" + "github.com/testmind-hq/caseforge/internal/sandbox" "github.com/testmind-hq/caseforge/internal/webhook" "github.com/testmind-hq/caseforge/internal/spec" "github.com/testmind-hq/caseforge/internal/tui" @@ -54,6 +58,7 @@ var ( genWithOracles bool genForce bool genAnnotationBatch int + genWithSandbox bool ) // allTechniqueNames is the canonical list used for --technique completion. @@ -110,6 +115,7 @@ func init() { genCmd.Flags().BoolVar(&genWithOracles, "with-oracles", false, "Mine response body constraints via LLM and inject as assertions (requires LLM)") genCmd.Flags().BoolVar(&genForce, "force", false, "Regenerate even when spec hash matches existing output") genCmd.Flags().IntVar(&genAnnotationBatch, "annotation-batch", 0, "Number of operations to annotate per LLM call (0 = one call per operation, recommended: 8–20)") + genCmd.Flags().BoolVar(&genWithSandbox, "with-sandbox", false, "Start a local sandbox server and run generated cases against it") _ = genCmd.MarkFlagRequired("spec") // Dynamic completion: --operations reads the spec and suggests operationIds. @@ -499,6 +505,13 @@ func runGen(cmd *cobra.Command, args []string) error { _ = ckptMgr.Delete() fmt.Fprintf(os.Stderr, "✓ Generated %d test cases → %s\n", len(cases), genOutput) + + if genWithSandbox { + if err := runWithSandbox(cmd, parsedSpec, genOutput); err != nil { + return fmt.Errorf("sandbox run: %w", err) + } + } + return nil } @@ -584,6 +597,45 @@ func filterByPriority(cases []schema.TestCase, minPriority string) []schema.Test return out } +// runWithSandbox starts a sandbox server, runs the generated cases against it via Hurl, +// prints results, and shuts the server down. Used by gen --with-sandbox. +func runWithSandbox(cmd *cobra.Command, ps *spec.ParsedSpec, casesDir string) error { + srv := sandbox.NewSandboxServer(ps, sandbox.Options{LogLevel: "silent"}) + if err := srv.Start("127.0.0.1", 0); err != nil { + return err + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(ctx) + }() + + fmt.Fprintf(cmd.ErrOrStderr(), "sandbox ready on http://%s\n", srv.Addr()) + + r := runner.NewHurlRunner() + vars := map[string]string{"base_url": "http://" + srv.Addr()} + result, err := r.Run(casesDir, vars) + if err != nil { + return fmt.Errorf("running cases: %w", err) + } + + out := cmd.OutOrStdout() + for _, c := range result.Cases { + if c.Passed { + color.New(color.FgGreen).Fprintf(out, " ✓ [%s] %s\n", c.ID, c.Title) + } else { + color.New(color.FgRed).Fprintf(out, " ✗ [%s] %s\n", c.ID, c.Title) + } + } + total := result.Passed + result.Failed + if result.Failed == 0 { + color.New(color.FgGreen).Fprintf(out, "✓ %d/%d sandbox tests passed\n", result.Passed, total) + return nil + } + color.New(color.FgYellow).Fprintf(out, "✗ %d/%d sandbox tests passed\n", result.Passed, total) + return fmt.Errorf("sandbox tests: %d/%d failed", result.Failed, total) +} + // splitTrimmed splits s on commas and trims whitespace from each token. func splitTrimmed(s string) []string { parts := strings.Split(s, ",") diff --git a/cmd/gen_e2e_test.go b/cmd/gen_e2e_test.go index a744ca1..2497068 100644 --- a/cmd/gen_e2e_test.go +++ b/cmd/gen_e2e_test.go @@ -38,6 +38,7 @@ func resetGenGlobals(t *testing.T) func() { genResume = false genForce = false genAnnotationBatch = 0 + genWithSandbox = false genTupleLevel = 2 genSeed = 0 } diff --git a/cmd/run.go b/cmd/run.go index d72c6e8..78326c5 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -22,7 +22,7 @@ var runCmd = &cobra.Command{ Long: `Run executes generated test case files against a live API. Supports hurl (.hurl files) and k6 (k6_tests.js) formats. -Use --target to set the API base URL (injected as BASE_URL variable). +Use --target to set the API base URL (injected as base_url for hurl, BASE_URL for k6). Use --output to write a run-report.json with structured results. Exit codes: @@ -44,7 +44,7 @@ func init() { _ = runCmd.MarkFlagRequired("cases") runCmd.Flags().StringArray("var", nil, "Variables as key=value (repeatable)") runCmd.Flags().StringVar(&runFormat, "format", "hurl", "Test runner format: hurl|k6") - runCmd.Flags().String("target", "", "API base URL, e.g. http://localhost:8080 (injected as BASE_URL)") + runCmd.Flags().String("target", "", "API base URL, e.g. http://localhost:8080 (injected as base_url for hurl, BASE_URL for k6)") runCmd.Flags().String("output", "", "Directory to write run-report.json (optional)") } @@ -54,19 +54,25 @@ func runRun(cmd *cobra.Command, _ []string) error { outputDir, _ := cmd.Flags().GetString("output") vars := runner.ParseVars(varFlags) - if target != "" { - if _, alreadySet := vars["BASE_URL"]; alreadySet { - fmt.Fprintf(cmd.ErrOrStderr(), "warning: --target overrides BASE_URL set via --var\n") - } - vars["BASE_URL"] = target - } var r runner.Runner switch runFormat { case "k6": r = runner.NewK6Runner() + if target != "" { + if _, alreadySet := vars["BASE_URL"]; alreadySet { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: --target overrides BASE_URL set via --var\n") + } + vars["BASE_URL"] = target + } default: r = runner.NewHurlRunner() + if target != "" { + if _, alreadySet := vars["base_url"]; alreadySet { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: --target overrides base_url set via --var\n") + } + vars["base_url"] = target + } } result, err := r.Run(runCases, vars) diff --git a/cmd/sandbox.go b/cmd/sandbox.go new file mode 100644 index 0000000..36628bc --- /dev/null +++ b/cmd/sandbox.go @@ -0,0 +1,80 @@ +// cmd/sandbox.go +package cmd + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" + "github.com/testmind-hq/caseforge/internal/sandbox" + "github.com/testmind-hq/caseforge/internal/spec" +) + +var ( + sandboxSpec string + sandboxPort int + sandboxHost string + sandboxLogLevel string + sandboxLogFile string + sandboxFormat string +) + +var sandboxCmd = &cobra.Command{ + Use: "sandbox", + Short: "Start a local mock API server from an OpenAPI spec", + Long: `Sandbox starts a local HTTP server that responds to requests described +by an OpenAPI spec. It generates realistic mock responses using a +strategy chain (example → schema → faker) and tracks stateful CRUD flows. + +Prints the listening address to stdout on startup. Stop with Ctrl-C. + +Examples: + caseforge sandbox --spec openapi.yaml + caseforge sandbox --spec openapi.yaml --port 8080 --log-level info --log-file sandbox.log`, + RunE: runSandbox, +} + +func init() { + rootCmd.AddCommand(sandboxCmd) + sandboxCmd.Flags().StringVar(&sandboxSpec, "spec", "", "OpenAPI spec file or URL (required)") + _ = sandboxCmd.MarkFlagRequired("spec") + sandboxCmd.Flags().IntVar(&sandboxPort, "port", 0, "Listen port (0 = random)") + sandboxCmd.Flags().StringVar(&sandboxHost, "host", "127.0.0.1", "Listen address") + sandboxCmd.Flags().StringVar(&sandboxLogLevel, "log-level", "info", "Log level: info|warn|error|silent") + sandboxCmd.Flags().StringVar(&sandboxLogFile, "log-file", "", "Append JSON logs to file (optional)") + sandboxCmd.Flags().StringVar(&sandboxFormat, "format", "auto", "Response strategy: auto|schema|faker") +} + +func runSandbox(cmd *cobra.Command, _ []string) error { + ps, err := spec.NewLoader().Load(sandboxSpec) + if err != nil { + return fmt.Errorf("loading spec: %w", err) + } + + opts := sandbox.Options{ + Host: sandboxHost, + Port: sandboxPort, + LogLevel: sandboxLogLevel, + LogFile: sandboxLogFile, + Format: sandboxFormat, + } + + srv := sandbox.NewSandboxServer(ps, opts) + if err := srv.Start(opts.Host, opts.Port); err != nil { + return err + } + + fmt.Fprintf(cmd.OutOrStdout(), "caseforge sandbox listening on http://%s\n", srv.Addr()) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + <-ctx.Done() + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return srv.Shutdown(shutdownCtx) +} diff --git a/internal/sandbox/generator.go b/internal/sandbox/generator.go new file mode 100644 index 0000000..15a2bb5 --- /dev/null +++ b/internal/sandbox/generator.go @@ -0,0 +1,101 @@ +// internal/sandbox/generator.go +package sandbox + +import ( + "fmt" + "strings" + + gofakeit "github.com/brianvoe/gofakeit/v7" + "github.com/testmind-hq/caseforge/internal/spec" +) + +// ResponseGenerator tries strategies in order and returns the first successful result. +type ResponseGenerator struct { + strategies []ResponseStrategy +} + +// newResponseGenerator builds the strategy chain for the given format option. +// "auto" = ExampleStrategy → SchemaStrategy → FakerStrategy +// "schema" = SchemaStrategy only +// "faker" = FakerStrategy only +func newResponseGenerator(format string) *ResponseGenerator { + var strategies []ResponseStrategy + switch format { + case "schema": + strategies = []ResponseStrategy{&schemaStrategy{}} + case "faker": + strategies = []ResponseStrategy{newFakerStrategy()} + default: // "auto" + strategies = []ResponseStrategy{ + &exampleStrategy{}, + &schemaStrategy{}, + newFakerStrategy(), + } + } + return &ResponseGenerator{strategies: strategies} +} + +// Generate produces a (statusCode, body) for the given operation. +// For write operations (POST/PUT/PATCH), the body is always a map[string]any +// with an "id" field guaranteed to be present. +func (g *ResponseGenerator) Generate(op *spec.Operation, pathParams map[string]string) (int, map[string]any) { + statusCode := successStatusCode(op) + var body any + for _, s := range g.strategies { + if result, ok := s.Generate(op, fmt.Sprintf("%d", statusCode)); ok { + body = result + break + } + } + // Normalize to map[string]any + m := toMap(body) + // Guarantee a non-empty id field is present for write operations so StateStore can key on it + if isWriteOp(op.Method) { + if v, ok := m["id"]; !ok || fmt.Sprintf("%v", v) == "" { + m["id"] = gofakeit.UUID() + } + } + return statusCode, m +} + +func successStatusCode(op *spec.Operation) int { + for _, code := range []string{"200", "201", "202"} { + if _, ok := op.Responses[code]; ok { + n := 0 + fmt.Sscanf(code, "%d", &n) + return n + } + } + return 200 +} + +func isWriteOp(method string) bool { + m := strings.ToUpper(method) + return m == "POST" || m == "PUT" || m == "PATCH" +} + +func toMap(v any) map[string]any { + if v == nil { + return map[string]any{} + } + if m, ok := v.(map[string]any); ok { + return m + } + return map[string]any{} +} + +// extractID looks for common ID field names in a response body, trying path-param names too. +func extractID(body map[string]any, opPath string) string { + candidates := []string{"id", "uuid", "ID"} + for _, seg := range strings.Split(opPath, "/") { + if strings.HasPrefix(seg, "{") && strings.HasSuffix(seg, "}") { + candidates = append(candidates, seg[1:len(seg)-1]) + } + } + for _, field := range candidates { + if val, ok := body[field]; ok { + return fmt.Sprintf("%v", val) + } + } + return "" +} diff --git a/internal/sandbox/handler.go b/internal/sandbox/handler.go new file mode 100644 index 0000000..97d5cdf --- /dev/null +++ b/internal/sandbox/handler.go @@ -0,0 +1,124 @@ +// internal/sandbox/handler.go +package sandbox + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/testmind-hq/caseforge/internal/spec" +) + +// resourceTypeFromPath extracts the primary resource name from an OpenAPI path. +// "/pets/{petId}" → "pets", "/users/{userId}/posts/{postId}" → "users_posts" +func resourceTypeFromPath(path string) string { + var segments []string + for _, seg := range strings.Split(strings.TrimPrefix(path, "/"), "/") { + if seg != "" && !strings.HasPrefix(seg, "{") { + segments = append(segments, seg) + } + } + if len(segments) == 0 { + return "resource" + } + return strings.Join(segments, "_") +} + +// idParamName returns the name of the first path parameter in an operation, or "id". +func idParamName(op *spec.Operation) string { + for _, p := range op.Parameters { + if p.In == "path" { + return p.Name + } + } + return "id" +} + +// hasPathParam reports whether the operation has any path parameters. +func hasPathParam(op *spec.Operation) bool { + for _, p := range op.Parameters { + if p.In == "path" { + return true + } + } + return false +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} + +func writeError(w http.ResponseWriter, status int, msg string) { + writeJSON(w, status, map[string]any{"error": msg, "status": status}) +} + +// makeHandler returns an http.HandlerFunc for the given operation. +func makeHandler(op *spec.Operation, store StateStore, gen *ResponseGenerator, logger *slog.Logger) http.HandlerFunc { + resourceType := resourceTypeFromPath(op.Path) + idParam := idParamName(op) + + return func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + method := strings.ToUpper(op.Method) + + switch method { + case "DELETE": + id := r.PathValue(idParam) + if _, ok := store.Read(resourceType, id); !ok { + writeError(w, http.StatusNotFound, "resource not found") + logger.Info("request", "method", r.Method, "path", r.URL.Path, "status", 404, "latency", time.Since(start)) + return + } + store.Delete(resourceType, id) + w.WriteHeader(http.StatusNoContent) + logger.Info("request", "method", r.Method, "path", r.URL.Path, "status", 204, "latency", time.Since(start)) + + case "GET": + if hasPathParam(op) { + id := r.PathValue(idParam) + obj, ok := store.Read(resourceType, id) + if !ok { + writeError(w, http.StatusNotFound, "resource not found") + logger.Info("request", "method", r.Method, "path", r.URL.Path, "status", 404, "latency", time.Since(start)) + return + } + writeJSON(w, http.StatusOK, obj) + logger.Info("request", "method", r.Method, "path", r.URL.Path, "status", 200, "latency", time.Since(start)) + } else { + list := store.List(resourceType) + writeJSON(w, http.StatusOK, list) + logger.Info("request", "method", r.Method, "path", r.URL.Path, "status", 200, "latency", time.Since(start)) + } + + default: // POST, PUT, PATCH + if r.Header.Get("Content-Type") != "" && !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + writeError(w, http.StatusUnsupportedMediaType, "Content-Type must be application/json") + logger.Info("request", "method", r.Method, "path", r.URL.Path, "status", 415, "latency", time.Since(start)) + return + } + pathParams := map[string]string{} + if hasPathParam(op) { + pathParams[idParam] = r.PathValue(idParam) + } + status, body := gen.Generate(op, pathParams) + id := extractID(body, op.Path) + if id == "" && hasPathParam(op) { + id = r.PathValue(idParam) + } + if id == "" { + id = fmt.Sprintf("%v", body["id"]) + } + if method == "POST" { + store.Write(resourceType, id, body) + } + w.Header().Set("X-Sandbox-ID", id) + writeJSON(w, status, body) + logger.Info("request", "method", r.Method, "path", r.URL.Path, "status", status, "latency", time.Since(start)) + } + } +} diff --git a/internal/sandbox/logging.go b/internal/sandbox/logging.go new file mode 100644 index 0000000..d290ba8 --- /dev/null +++ b/internal/sandbox/logging.go @@ -0,0 +1,103 @@ +// internal/sandbox/logging.go +package sandbox + +import ( + "context" + "errors" + "io" + "log/slog" + "os" +) + +// multiHandler fans log records out to multiple slog.Handler implementations. +type multiHandler struct { + handlers []slog.Handler +} + +func (m *multiHandler) Enabled(ctx context.Context, l slog.Level) bool { + for _, h := range m.handlers { + if h.Enabled(ctx, l) { + return true + } + } + return false +} + +func (m *multiHandler) Handle(ctx context.Context, r slog.Record) error { + var errs []error + for _, h := range m.handlers { + if h.Enabled(ctx, r.Level) { + if err := h.Handle(ctx, r.Clone()); err != nil { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +func (m *multiHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + hs := make([]slog.Handler, len(m.handlers)) + for i, h := range m.handlers { + hs[i] = h.WithAttrs(attrs) + } + return &multiHandler{handlers: hs} +} + +func (m *multiHandler) WithGroup(name string) slog.Handler { + hs := make([]slog.Handler, len(m.handlers)) + for i, h := range m.handlers { + hs[i] = h.WithGroup(name) + } + return &multiHandler{handlers: hs} +} + +// buildLogger constructs a slog.Logger from Options. +// Returns the logger and a cleanup function (closes the log file if opened). +func buildLogger(opts Options) (*slog.Logger, func()) { + level := parseLevel(opts.logLevelOrDefault()) + + var handlers []slog.Handler + var closers []io.Closer + + if opts.logLevelOrDefault() != "silent" { + handlers = append(handlers, slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})) + } + + if opts.LogFile != "" { + f, err := os.OpenFile(opts.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err == nil { + handlers = append(handlers, slog.NewJSONHandler(f, &slog.HandlerOptions{Level: level})) + closers = append(closers, f) + } + } + + var h slog.Handler + switch len(handlers) { + case 0: + h = slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError + 100}) + case 1: + h = handlers[0] + default: + h = &multiHandler{handlers: handlers} + } + + cleanup := func() { + for _, c := range closers { + _ = c.Close() + } + } + return slog.New(h), cleanup +} + +func parseLevel(level string) slog.Level { + switch level { + case "warn": + return slog.LevelWarn + case "error": + return slog.LevelError + case "silent": + return slog.LevelError + 100 // effectively disabled + default: // "info" + return slog.LevelInfo + } +} diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go new file mode 100644 index 0000000..595e969 --- /dev/null +++ b/internal/sandbox/sandbox.go @@ -0,0 +1,32 @@ +// internal/sandbox/sandbox.go +package sandbox + +// Options controls the sandbox server's runtime behaviour. +type Options struct { + Host string // listen address, default "127.0.0.1" + Port int // 0 = OS-assigned random port + LogLevel string // "info" | "warn" | "error" | "silent" + LogFile string // path to append JSON log; "" = no file + Format string // "auto" | "schema" | "faker" +} + +func (o Options) hostOrDefault() string { + if o.Host == "" { + return "127.0.0.1" + } + return o.Host +} + +func (o Options) logLevelOrDefault() string { + if o.LogLevel == "" { + return "info" + } + return o.LogLevel +} + +func (o Options) formatOrDefault() string { + if o.Format == "" { + return "auto" + } + return o.Format +} diff --git a/internal/sandbox/server.go b/internal/sandbox/server.go new file mode 100644 index 0000000..4ba8d76 --- /dev/null +++ b/internal/sandbox/server.go @@ -0,0 +1,63 @@ +// internal/sandbox/server.go +package sandbox + +import ( + "context" + "fmt" + "net" + "net/http" + "strings" + + "github.com/testmind-hq/caseforge/internal/spec" +) + +// SandboxServer is a local HTTP server that mocks an OpenAPI-described API. +type SandboxServer struct { + srv *http.Server + addr string // resolved after Start() + logCleanup func() // closes log file(s) opened by buildLogger +} + +// NewSandboxServer builds (but does not start) a SandboxServer for the given spec. +func NewSandboxServer(ps *spec.ParsedSpec, opts Options) *SandboxServer { + logger, cleanup := buildLogger(opts) + + gen := newResponseGenerator(opts.formatOrDefault()) + store := newMemStateStore() + + mux := http.NewServeMux() + for _, op := range ps.Operations { + pattern := fmt.Sprintf("%s %s", strings.ToUpper(op.Method), op.Path) + mux.HandleFunc(pattern, makeHandler(op, store, gen, logger)) + } + + srv := &http.Server{Handler: mux} + + return &SandboxServer{ + srv: srv, + logCleanup: cleanup, + } +} + +// Start listens on the given host:port and begins serving in a goroutine. +// When port is 0, the OS assigns a random available port. +// Addr() returns the actual address after Start returns. +func (s *SandboxServer) Start(host string, port int) error { + addr := fmt.Sprintf("%s:%d", host, port) + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("sandbox listen: %w", err) + } + s.addr = ln.Addr().String() + go s.srv.Serve(ln) //nolint:errcheck + return nil +} + +// Addr returns "host:port" of the listening server. Call after Start. +func (s *SandboxServer) Addr() string { return s.addr } + +// Shutdown gracefully stops the server within the given context deadline. +func (s *SandboxServer) Shutdown(ctx context.Context) error { + defer s.logCleanup() + return s.srv.Shutdown(ctx) +} diff --git a/internal/sandbox/server_test.go b/internal/sandbox/server_test.go new file mode 100644 index 0000000..15fb135 --- /dev/null +++ b/internal/sandbox/server_test.go @@ -0,0 +1,160 @@ +// internal/sandbox/server_test.go +package sandbox + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testmind-hq/caseforge/internal/spec" +) + +// petstoreSpec returns a minimal ParsedSpec modelling GET/POST /pets and GET/DELETE /pets/{petId}. +func petstoreSpec() *spec.ParsedSpec { + idSchema := &spec.Schema{Type: "string", Format: "uuid"} + nameSchema := &spec.Schema{Type: "string"} + petSchema := &spec.Schema{ + Type: "object", + Properties: map[string]*spec.Schema{"id": idSchema, "name": nameSchema}, + } + jsonContent := func(s *spec.Schema) map[string]*spec.MediaType { + return map[string]*spec.MediaType{"application/json": {Schema: s}} + } + return &spec.ParsedSpec{ + Operations: []*spec.Operation{ + { + OperationID: "listPets", + Method: "GET", + Path: "/pets", + Responses: map[string]*spec.Response{ + "200": {Content: jsonContent(&spec.Schema{Type: "array", Items: petSchema})}, + }, + }, + { + OperationID: "createPet", + Method: "POST", + Path: "/pets", + Responses: map[string]*spec.Response{ + "201": {Content: jsonContent(petSchema)}, + }, + }, + { + OperationID: "showPetById", + Method: "GET", + Path: "/pets/{petId}", + Parameters: []*spec.Parameter{ + {Name: "petId", In: "path", Required: true, Schema: &spec.Schema{Type: "string"}}, + }, + Responses: map[string]*spec.Response{ + "200": {Content: jsonContent(petSchema)}, + "404": {Description: "Not found"}, + }, + }, + { + OperationID: "deletePet", + Method: "DELETE", + Path: "/pets/{petId}", + Parameters: []*spec.Parameter{ + {Name: "petId", In: "path", Required: true, Schema: &spec.Schema{Type: "string"}}, + }, + Responses: map[string]*spec.Response{ + "204": {Description: "Deleted"}, + "404": {Description: "Not found"}, + }, + }, + }, + } +} + +// newTestMux builds an http.ServeMux from a ParsedSpec exactly as NewSandboxServer does, +// but returns it directly so httptest.NewServer can wrap it. +func newTestMux(ps *spec.ParsedSpec) http.Handler { + logger, _ := buildLogger(Options{LogLevel: "silent"}) + gen := newResponseGenerator("auto") + store := newMemStateStore() + mux := http.NewServeMux() + for _, op := range ps.Operations { + pattern := strings.ToUpper(op.Method) + " " + op.Path + mux.HandleFunc(pattern, makeHandler(op, store, gen, logger)) + } + return mux +} + +func TestSandbox_ListPets_Empty(t *testing.T) { + ts := httptest.NewServer(newTestMux(petstoreSpec())) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/pets") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var list []any + require.NoError(t, json.NewDecoder(resp.Body).Decode(&list)) + assert.NotNil(t, list) +} + +func TestSandbox_CRUD_Flow(t *testing.T) { + ts := httptest.NewServer(newTestMux(petstoreSpec())) + defer ts.Close() + + // POST → 201 + body := bytes.NewBufferString(`{"name":"Fido"}`) + resp, err := http.Post(ts.URL+"/pets", "application/json", body) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + data, _ := io.ReadAll(resp.Body) + var created map[string]any + require.NoError(t, json.Unmarshal(data, &created)) + id, ok := created["id"].(string) + require.True(t, ok, "POST response must include id field") + require.NotEmpty(t, id) + + // GET /{id} → 200 + resp2, err := http.Get(ts.URL + "/pets/" + id) + require.NoError(t, err) + defer resp2.Body.Close() + assert.Equal(t, http.StatusOK, resp2.StatusCode) + + // DELETE /{id} → 204 + req, _ := http.NewRequest(http.MethodDelete, ts.URL+"/pets/"+id, nil) + resp3, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp3.Body.Close() + assert.Equal(t, http.StatusNoContent, resp3.StatusCode) + + // GET /{id} after delete → 404 + resp4, err := http.Get(ts.URL + "/pets/" + id) + require.NoError(t, err) + defer resp4.Body.Close() + assert.Equal(t, http.StatusNotFound, resp4.StatusCode) +} + +func TestSandbox_WrongContentType_Returns415(t *testing.T) { + ts := httptest.NewServer(newTestMux(petstoreSpec())) + defer ts.Close() + + resp, err := http.Post(ts.URL+"/pets", "text/plain", strings.NewReader("name=Fido")) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusUnsupportedMediaType, resp.StatusCode) +} + +func TestSandbox_Start_Addr(t *testing.T) { + ps := petstoreSpec() + srv := NewSandboxServer(ps, Options{LogLevel: "silent"}) + err := srv.Start("127.0.0.1", 0) + require.NoError(t, err) + assert.NotEmpty(t, srv.Addr()) + + // Shutdown to release the port + _ = srv.Shutdown(t.Context()) +} diff --git a/internal/sandbox/state.go b/internal/sandbox/state.go new file mode 100644 index 0000000..a13de19 --- /dev/null +++ b/internal/sandbox/state.go @@ -0,0 +1,63 @@ +// internal/sandbox/state.go +package sandbox + +import "sync" + +// StateStore tracks in-memory resource objects for stateful CRUD flows. +type StateStore interface { + Write(resourceType, id string, body map[string]any) + Read(resourceType, id string) (map[string]any, bool) + Delete(resourceType, id string) + List(resourceType string) []map[string]any +} + +type memStateStore struct { + mu sync.RWMutex + store map[string]map[string]map[string]any // resourceType → id → object +} + +func newMemStateStore() *memStateStore { + return &memStateStore{store: make(map[string]map[string]map[string]any)} +} + +func (m *memStateStore) Write(resourceType, id string, body map[string]any) { + m.mu.Lock() + defer m.mu.Unlock() + if m.store[resourceType] == nil { + m.store[resourceType] = make(map[string]map[string]any) + } + m.store[resourceType][id] = body +} + +func (m *memStateStore) Read(resourceType, id string) (map[string]any, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + if rt, ok := m.store[resourceType]; ok { + if obj, ok := rt[id]; ok { + return obj, true + } + } + return nil, false +} + +func (m *memStateStore) Delete(resourceType, id string) { + m.mu.Lock() + defer m.mu.Unlock() + if rt, ok := m.store[resourceType]; ok { + delete(rt, id) + } +} + +func (m *memStateStore) List(resourceType string) []map[string]any { + m.mu.RLock() + defer m.mu.RUnlock() + rt, ok := m.store[resourceType] + if !ok { + return []map[string]any{} + } + out := make([]map[string]any, 0, len(rt)) + for _, v := range rt { + out = append(out, v) + } + return out +} diff --git a/internal/sandbox/state_test.go b/internal/sandbox/state_test.go new file mode 100644 index 0000000..b87647f --- /dev/null +++ b/internal/sandbox/state_test.go @@ -0,0 +1,59 @@ +// internal/sandbox/state_test.go +package sandbox + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStateStore_WriteReadDelete(t *testing.T) { + s := newMemStateStore() + + s.Write("pets", "1", map[string]any{"id": "1", "name": "Fido"}) + + obj, ok := s.Read("pets", "1") + require.True(t, ok) + assert.Equal(t, "Fido", obj["name"]) + + s.Delete("pets", "1") + _, ok = s.Read("pets", "1") + assert.False(t, ok) +} + +func TestStateStore_List(t *testing.T) { + s := newMemStateStore() + s.Write("pets", "1", map[string]any{"id": "1"}) + s.Write("pets", "2", map[string]any{"id": "2"}) + + list := s.List("pets") + assert.Len(t, list, 2) +} + +func TestStateStore_List_Empty(t *testing.T) { + s := newMemStateStore() + list := s.List("pets") + assert.NotNil(t, list) + assert.Len(t, list, 0) +} + +func TestStateStore_ConcurrentReadWrite(t *testing.T) { + s := newMemStateStore() + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(2) + id := fmt.Sprintf("%d", i) + go func(id string) { + defer wg.Done() + s.Write("pets", id, map[string]any{"id": id}) + }(id) + go func(id string) { + defer wg.Done() + _, _ = s.Read("pets", id) + }(id) + } + wg.Wait() +} diff --git a/internal/sandbox/strategies.go b/internal/sandbox/strategies.go new file mode 100644 index 0000000..2b3bc1f --- /dev/null +++ b/internal/sandbox/strategies.go @@ -0,0 +1,148 @@ +// internal/sandbox/strategies.go +package sandbox + +import ( + "github.com/testmind-hq/caseforge/internal/datagen" + "github.com/testmind-hq/caseforge/internal/spec" +) + +// ResponseStrategy generates a response body for an operation and status code. +// Returns (body, true) when it can produce a value; (nil, false) to pass to next strategy. +type ResponseStrategy interface { + Name() string + Generate(op *spec.Operation, statusCode string) (any, bool) +} + +// successResponse returns the first 2xx response found, preferring 200 → 201 → 202. +func successResponse(op *spec.Operation) *spec.Response { + for _, code := range []string{"200", "201", "202"} { + if resp, ok := op.Responses[code]; ok { + return resp + } + } + return nil +} + +// --- ExampleStrategy --- + +type exampleStrategy struct{} + +func (e *exampleStrategy) Name() string { return "example" } + +func (e *exampleStrategy) Generate(op *spec.Operation, statusCode string) (any, bool) { + resp, ok := op.Responses[statusCode] + if !ok { + resp = successResponse(op) + if resp == nil { + return nil, false + } + } + mt, ok := resp.Content["application/json"] + if !ok { + return nil, false + } + if mt.Example != nil { + return mt.Example, true + } + for _, ex := range mt.Examples { + if ex != nil && ex.Value != nil { + return ex.Value, true + } + } + return nil, false +} + +// --- SchemaStrategy --- + +type schemaStrategy struct{} + +func (s *schemaStrategy) Name() string { return "schema" } + +func (s *schemaStrategy) Generate(op *spec.Operation, statusCode string) (any, bool) { + resp, ok := op.Responses[statusCode] + if !ok { + resp = successResponse(op) + if resp == nil { + return nil, false + } + } + mt, ok := resp.Content["application/json"] + if !ok || mt == nil || mt.Schema == nil { + return nil, false + } + return schemaZeroValue(mt.Schema), true +} + +func schemaZeroValue(s *spec.Schema) any { + if s == nil { + return nil + } + switch s.Type { + case "object": + obj := map[string]any{} + for name, prop := range s.Properties { + obj[name] = schemaZeroValue(prop) + } + return obj + case "array": + if s.Items != nil { + return []any{schemaZeroValue(s.Items)} + } + return []any{} + case "integer": + return int64(0) + case "number": + return float64(0) + case "boolean": + return false + default: // "string" and unknown + return "" + } +} + +// --- FakerStrategy --- + +type fakerStrategy struct { + gen *datagen.Generator +} + +func newFakerStrategy() *fakerStrategy { + return &fakerStrategy{gen: datagen.NewGenerator(nil)} +} + +func (f *fakerStrategy) Name() string { return "faker" } + +func (f *fakerStrategy) Generate(op *spec.Operation, statusCode string) (any, bool) { + resp, ok := op.Responses[statusCode] + if !ok { + resp = successResponse(op) + } + if resp == nil { + return map[string]any{}, true + } + mt, ok := resp.Content["application/json"] + if !ok || mt == nil { + return map[string]any{}, true + } + return f.generateFromSchema(mt.Schema), true +} + +func (f *fakerStrategy) generateFromSchema(s *spec.Schema) any { + if s == nil { + return map[string]any{} + } + if s.Type == "object" { + obj := map[string]any{} + for name, prop := range s.Properties { + obj[name] = f.gen.Generate(prop, name) + } + return obj + } + if s.Type == "array" { + if s.Items != nil { + return []any{f.generateFromSchema(s.Items)} + } + return []any{} + } + return f.gen.Generate(s, "") +} diff --git a/internal/sandbox/strategies_test.go b/internal/sandbox/strategies_test.go new file mode 100644 index 0000000..18d52c7 --- /dev/null +++ b/internal/sandbox/strategies_test.go @@ -0,0 +1,98 @@ +// internal/sandbox/strategies_test.go +package sandbox + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/testmind-hq/caseforge/internal/spec" +) + +func opWithExample() *spec.Operation { + return &spec.Operation{ + Method: "GET", + Path: "/pets", + Responses: map[string]*spec.Response{ + "200": { + Content: map[string]*spec.MediaType{ + "application/json": { + Example: map[string]any{"id": "abc", "name": "Fido"}, + }, + }, + }, + }, + } +} + +func opWithSchema() *spec.Operation { + return &spec.Operation{ + Method: "GET", + Path: "/pets/1", + Responses: map[string]*spec.Response{ + "200": { + Content: map[string]*spec.MediaType{ + "application/json": { + Schema: &spec.Schema{ + Type: "object", + Properties: map[string]*spec.Schema{ + "id": {Type: "string", Format: "uuid"}, + "name": {Type: "string"}, + }, + }, + }, + }, + }, + }, + } +} + +func opNoResponse() *spec.Operation { + return &spec.Operation{ + Method: "DELETE", + Path: "/pets/1", + Responses: map[string]*spec.Response{"204": {Description: "Deleted"}}, + } +} + +func TestExampleStrategy_UsesExample(t *testing.T) { + s := &exampleStrategy{} + out, ok := s.Generate(opWithExample(), "200") + assert.True(t, ok) + m, _ := out.(map[string]any) + assert.Equal(t, "Fido", m["name"]) +} + +func TestExampleStrategy_NoExample(t *testing.T) { + s := &exampleStrategy{} + _, ok := s.Generate(opWithSchema(), "200") + assert.False(t, ok) +} + +func TestSchemaStrategy_ProducesObject(t *testing.T) { + s := &schemaStrategy{} + out, ok := s.Generate(opWithSchema(), "200") + assert.True(t, ok) + m, _ := out.(map[string]any) + assert.Contains(t, m, "id") + assert.Contains(t, m, "name") +} + +func TestSchemaStrategy_NoSchema(t *testing.T) { + s := &schemaStrategy{} + _, ok := s.Generate(opNoResponse(), "204") + assert.False(t, ok) +} + +func TestFakerStrategy_AlwaysProduces(t *testing.T) { + s := newFakerStrategy() + out, ok := s.Generate(opWithSchema(), "200") + assert.True(t, ok) + assert.NotNil(t, out) +} + +func TestFakerStrategy_NoSchema(t *testing.T) { + s := newFakerStrategy() + out, ok := s.Generate(opNoResponse(), "204") + assert.True(t, ok) // faker always returns true + assert.NotNil(t, out) +} diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index e9801c4..ef6379e 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -55,6 +55,12 @@ exits_with() { [ -n "$VERBOSE" ] && cat "$WORKDIR/out" fi } +skip() { + local id="$1"; local desc="$2"; local reason="${3:-}" + log " ⏭ SKIP $id: $desc${reason:+ [$reason]}" +} +# random_port: portable random port in [20000, 39999] +random_port() { echo $(( (RANDOM % 20000) + 20000 )); } # ------------------------------------------------------- # Fixtures @@ -1526,6 +1532,99 @@ run AT-237 "generate --no-ai works with bedrock config" \ '$BIN' gen --spec '$WORKDIR/petstore.yaml' --no-ai --output '$WORKDIR/at237/cases'" echo "" +# ------------------------------------------------------- +# AT-301 – AT-303: sandbox command +# ------------------------------------------------------- +echo "# AT-301 – AT-303: sandbox" + +# AT-301: sandbox starts and GET /pets returns 200 +run "AT-301" "sandbox GET /pets returns 200" \ + "PORT=\$(random_port) + '$BIN' sandbox --spec '$WORKDIR/petstore.yaml' --port \$PORT & + SBX_PID=\$! + for i in \$(seq 1 20); do curl -sf http://127.0.0.1:\$PORT/pets > /dev/null 2>&1 && break; sleep 0.1; done + STATUS=\$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:\$PORT/pets) + kill \$SBX_PID 2>/dev/null; wait \$SBX_PID 2>/dev/null + [ \"\$STATUS\" = '200' ]" + +# AT-302: full CRUD flow POST→GET→DELETE +run "AT-302" "sandbox CRUD flow POST→GET→DELETE" \ + "PORT=\$(random_port) + '$BIN' sandbox --spec '$WORKDIR/petstore.yaml' --port \$PORT & + SBX_PID=\$! + for i in \$(seq 1 20); do curl -sf http://127.0.0.1:\$PORT/pets > /dev/null 2>&1 && break; sleep 0.1; done + POST_STATUS=\$(curl -s -o /tmp/at302body.json -w '%{http_code}' -X POST -H 'Content-Type: application/json' -d '{\"name\":\"Fido\"}' http://127.0.0.1:\$PORT/pets) + PET_ID=\$(python3 -c \"import json,sys; d=json.load(open('/tmp/at302body.json')); print(d.get('id',''))\" 2>/dev/null) + if [ -z \"\$PET_ID\" ]; then PET_ID=\$(curl -s -D - -X POST -H 'Content-Type: application/json' -d '{\"name\":\"Rex\"}' http://127.0.0.1:\$PORT/pets | grep -i x-sandbox-id | awk '{print \$2}' | tr -d '\r'); fi + GET_STATUS=\$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:\$PORT/pets/\$PET_ID) + DEL_STATUS=\$(curl -s -o /dev/null -w '%{http_code}' -X DELETE http://127.0.0.1:\$PORT/pets/\$PET_ID) + kill \$SBX_PID 2>/dev/null; wait \$SBX_PID 2>/dev/null + [ \"\$POST_STATUS\" = '201' ] && [ \"\$GET_STATUS\" = '200' ] && [ \"\$DEL_STATUS\" = '204' ]" + +# AT-303: gen --with-sandbox end-to-end (skipped when hurl is not installed) +# Uses a minimal spec with no 4xx responses and --technique equivalence_partitioning +# so only happy-path cases are generated — cases the sandbox can satisfy. +if command -v hurl > /dev/null 2>&1; then + cat > "$WORKDIR/sandbox-smoke.yaml" << 'SPECEOF' +openapi: "3.0.0" +info: + title: Sandbox Smoke Test + version: "1.0" +paths: + /items: + get: + operationId: listItems + responses: + "200": + description: OK + content: + application/json: + schema: + type: array + items: + type: object + properties: + id: + type: string + name: + type: string + post: + operationId: createItem + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + name: + type: string + responses: + "201": + description: Created + content: + application/json: + schema: + type: object + properties: + id: + type: string + name: + type: string +SPECEOF + run "AT-303" "gen --with-sandbox runs happy-path cases against sandbox and exits 0" \ + "mkdir -p '$WORKDIR/at303' && \ + NOPROV='$WORKDIR/at303/.caseforge.yaml' && \ + printf 'ai:\n provider: noop\n' > \"\$NOPROV\" && \ + '$BIN' gen --config \"\$NOPROV\" --spec '$WORKDIR/sandbox-smoke.yaml' --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output '$WORKDIR/at303/cases' --with-sandbox" +else + skip "AT-303" "gen --with-sandbox end-to-end" "hurl not installed" +fi + +echo "" + # ------------------------------------------------------- # Summary # -------------------------------------------------------