diff --git a/README.md b/README.md index 78dc842..97cb4e4 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,10 @@ test spec also contains these fields: the name of the environment variable to read into the named variable. * `assert`: (optional) an object describing the conditions that will be asserted about the test action. +* `assert.require`: (optional) a boolean indicating whether a failed assertion + will cause the test scenario's execution to stop. The default behaviour of + `gdt` is to continue execution of subsequent test specs in a test scenario when + an assertion fails. * `assert.exit-code`: (optional) an integer with the expected exit code from the executed command. The default successful exit code is 0 and therefore you do not need to specify this if you expect a successful exit code. diff --git a/api/result.go b/api/result.go index 444df11..66c0dd4 100644 --- a/api/result.go +++ b/api/result.go @@ -16,6 +16,9 @@ package api // returned in the Result and the `Scenario.Run` method injects that // information into the context that is supplied to the next Spec's `Run`. type Result struct { + // stopOnFail is an indication to the scenario that if there are any + // failures, the scenario should not proceed with test execution. + stopOnFail bool // failures is the collection of error messages from assertion failures // that occurred during Eval(). These are *not* `gdterrors.RuntimeError`. failures []error @@ -39,6 +42,12 @@ func (r *Result) Data() map[string]any { return r.data } +// StopOnFail returns true if the test spec indicates that a failure of +// assertion should stop the execution of the test scenario. +func (r *Result) StopOnFail() bool { + return r.stopOnFail +} + // Failed returns true if any assertion failed during Eval(), false otherwise. func (r *Result) Failed() bool { return len(r.failures) > 0 @@ -95,6 +104,14 @@ func WithData(key string, val any) ResultModifier { } } +// WithStopOnFail sets the stopOnFail value for the test spec result. +// failures +func WithStopOnFail(val bool) ResultModifier { + return func(r *Result) { + r.stopOnFail = val + } +} + // WithFailures modifies the Result the supplied collection of assertion // failures func WithFailures(failures ...error) ResultModifier { diff --git a/parse/error.go b/parse/error.go index 8ea2c30..7c2cde7 100644 --- a/parse/error.go +++ b/parse/error.go @@ -179,6 +179,16 @@ func ExpectedScalarOrMapAt(node *yaml.Node) error { } } +// ExpectedBoolAt returns a parse error indicating a boolean value was expected +// and annotated with the line/column of the supplied YAML node. +func ExpectedBoolAt(node *yaml.Node) error { + return &Error{ + Line: node.Line, + Column: node.Column, + Message: "expected boolean value", + } +} + // ExpectedTimeoutAt returns an ErrExpectedTimeout error annotated // with the line/column of the supplied YAML node. func ExpectedTimeoutAt(node *yaml.Node) error { diff --git a/plugin/exec/assertions.go b/plugin/exec/assertions.go index 1c867e0..f66ee98 100644 --- a/plugin/exec/assertions.go +++ b/plugin/exec/assertions.go @@ -18,6 +18,9 @@ import ( // Expect contains the assertions about an Exec Spec's actions type Expect struct { + // Require indicates that any failed assertion should stop the execution of + // the test scenario in which the test spec is contained. + Require bool `yaml:"require,omitempty"` // ExitCode is the expected exit code for the executed command. The default // (0) is the universal successful exit code, so you only need to set this // if you expect a non-successful result from executing the command. diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go index e3695a8..bfa574e 100644 --- a/plugin/exec/eval.go +++ b/plugin/exec/eval.go @@ -47,5 +47,12 @@ func (s *Spec) Eval( } } } - return api.NewResult(api.WithFailures(a.Failures()...)), nil + stopOnFail := false + if s.Assert != nil { + stopOnFail = s.Assert.Require + } + return api.NewResult( + api.WithStopOnFail(stopOnFail), + api.WithFailures(a.Failures()...), + ), nil } diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go index 6cbd648..ce3b041 100644 --- a/plugin/exec/eval_test.go +++ b/plugin/exec/eval_test.go @@ -463,3 +463,47 @@ func TestVar(t *testing.T) { err = s.Run(ctx, t) require.Nil(err) } + +func TestFailStopOnFail(t *testing.T) { + if !*failFlag { + t.Skip("skipping without -fail flag") + } + require := require.New(t) + + fp := filepath.Join("testdata", "stop-on-fail.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader( + f, + scenario.WithPath(fp), + ) + require.Nil(err) + require.NotNil(s) + + ctx := gdtcontext.New(gdtcontext.WithDebug()) + err = s.Run(ctx, t) + require.Nil(err) +} + +func TestStopOnFail(t *testing.T) { + require := require.New(t) + target := os.Args[0] + failArgs := []string{ + "-test.v", + "-test.run=FailStopOnFail", + "-fail", + } + outerr, err := exec.Command(target, failArgs...).CombinedOutput() + + // The test should have failed... + require.NotNil(err) + + debugout := string(outerr) + require.Contains(debugout, "assertion failed: not in: expected stdout to contain 1234") + // first test spec does not contain the `require: true` field and so the + // second test spec should execute (and fail its assertion) + require.Contains(debugout, "assertion failed: not in: expected stdout to contain 24") + // The third test spec should NOT have been executed... + require.NotContains(debugout, "[gdt] [stop-on-fail/2] exec: stdout: 24") +} diff --git a/plugin/exec/parse.go b/plugin/exec/parse.go index 44e9515..48075b5 100644 --- a/plugin/exec/parse.go +++ b/plugin/exec/parse.go @@ -123,6 +123,16 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error { return err } s.Assert = e + case "require": + if valNode.Kind != yaml.MappingNode { + return parse.ExpectedMapAt(valNode) + } + var e *Expect + if err := valNode.Decode(&e); err != nil { + return err + } + e.Require = true + s.Assert = e case "on": if valNode.Kind != yaml.MappingNode { return parse.ExpectedMapAt(valNode) @@ -168,6 +178,16 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error { key := keyNode.Value valNode := node.Content[i+1] switch key { + case "require", "stop-on-fail", "stop_on_fail", "stop.on.fail", + "fail-stop", "fail.stop", "fail_stop": + if valNode.Kind != yaml.ScalarNode { + return parse.ExpectedScalarAt(valNode) + } + req, err := strconv.ParseBool(valNode.Value) + if err != nil { + return parse.ExpectedBoolAt(valNode) + } + e.Require = req case "exit_code", "exit-code": if valNode.Kind != yaml.ScalarNode { return parse.ExpectedScalarAt(valNode) diff --git a/plugin/exec/spec.go b/plugin/exec/spec.go index 4b21c3d..2cb511f 100644 --- a/plugin/exec/spec.go +++ b/plugin/exec/spec.go @@ -13,6 +13,10 @@ import ( type Spec struct { api.Spec Action + // Require is an object containing the conditions that the Spec will + // assert. If any condition fails, the test scenario execution will stop + // and be marked as failed. + Require *Expect `yaml:"require,omitempty"` // Assert is an object containing the conditions that the Spec will assert. Assert *Expect `yaml:"assert,omitempty"` // On is an object containing actions to take upon certain conditions. diff --git a/plugin/exec/testdata/stop-on-fail.yaml b/plugin/exec/testdata/stop-on-fail.yaml new file mode 100644 index 0000000..dadb564 --- /dev/null +++ b/plugin/exec/testdata/stop-on-fail.yaml @@ -0,0 +1,17 @@ +name: stop-on-fail +description: | + a scenario that tests that a failed assertion for a stop-on-fail + test spec prevents subsequent test specs from being executed. +tests: + - exec: echo "meaning of life" + assert: + out: + is: 1234 + # This test should be executed because we do not use the require: true. + - exec: echo 42 + assert: + require: true + out: + is: 24 + # This test should not be executed because of the use of require: true above + - exec: echo 24 diff --git a/scenario/run.go b/scenario/run.go index 629b872..d9920f9 100644 --- a/scenario/run.go +++ b/scenario/run.go @@ -108,6 +108,7 @@ func (s *Scenario) runExternal(ctx context.Context, run *run.Run) error { scenCleanups := []func(){} scenOK := true +outer: for idx, t := range s.Tests { tu := testunit.New( ctx, @@ -134,11 +135,15 @@ func (s *Scenario) runExternal(ctx context.Context, run *run.Run) error { if res.HasData() { ctx = gdtcontext.SetRun(ctx, res.Data()) } - if len(res.Failures()) > 0 { - tu.FailNow() - } else { - tu.Finish() // necessary for elapsed timer to stop + for _, fail := range res.Failures() { + if res.StopOnFail() { + tu.Fatal(fail) + run.StoreResult(idx, s.Path, tu, res) + break outer + } + tu.Error(fail) } + tu.Finish() // necessary for elapsed timer to stop scenOK = scenOK && !tu.Failed() run.StoreResult(idx, s.Path, tu, res) @@ -220,7 +225,10 @@ func (s *Scenario) runGo(ctx context.Context, t *testing.T) error { } for _, fail := range res.Failures() { - tt.Fatal(fail) + if res.StopOnFail() { + tt.Fatal(fail) + } + tt.Error(fail) } } })