diff --git a/CHANGELOG.md b/CHANGELOG.md index bc4a3ea..081c6ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change LOG +**2017-05-04** +- added **--strict** option in order to fail suite when there are pending + or undefined steps. By default, suite passes and treats pending or + undefined steps as TODOs. + **2017-04-29** - **v0.7.0** - added support for nested steps. From now on, it is possible to return **godog.Steps** instead of an **error** in the step definition func. diff --git a/flags.go b/flags.go index 9740e14..71bea29 100644 --- a/flags.go +++ b/flags.go @@ -52,6 +52,7 @@ func FlagSet(opt *Options) *flag.FlagSet { set.BoolVar(&opt.ShowStepDefinitions, "definitions", false, "Print all available step definitions.") set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.") set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.") + set.BoolVar(&opt.Strict, "strict", false, "Fail suite when there are pending or undefined steps.") set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.") set.Var(&randomSeed{&opt.Randomize}, "random", descRandomOption) set.Usage = usage(set, opt.Output) diff --git a/options.go b/options.go index 36710c2..064b414 100644 --- a/options.go +++ b/options.go @@ -35,6 +35,9 @@ type Options struct { // Stops on the first failure StopOnFailure bool + // Fail suite when there are pending or undefined steps + Strict bool + // Forces ansi color stripping NoColors bool diff --git a/run.go b/run.go index 9e934b3..7ab39e8 100644 --- a/run.go +++ b/run.go @@ -12,11 +12,11 @@ import ( type initializer func(*Suite) type runner struct { - randomSeed int64 - stopOnFailure bool - features []*feature - fmt Formatter - initializer initializer + randomSeed int64 + stopOnFailure, strict bool + features []*feature + fmt Formatter + initializer initializer } func (r *runner) concurrent(rate int) (failed bool) { @@ -34,6 +34,7 @@ func (r *runner) concurrent(rate int) (failed bool) { fmt: r.fmt, randomSeed: r.randomSeed, stopOnFailure: r.stopOnFailure, + strict: r.strict, features: []*feature{feat}, } r.initializer(suite) @@ -59,6 +60,7 @@ func (r *runner) run() bool { fmt: r.fmt, randomSeed: r.randomSeed, stopOnFailure: r.stopOnFailure, + strict: r.strict, features: r.features, } r.initializer(suite) @@ -116,6 +118,7 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt features: features, randomSeed: opt.Randomize, stopOnFailure: opt.StopOnFailure, + strict: opt.Strict, } // store chosen seed in environment, so it could be seen in formatter summary report diff --git a/run_test.go b/run_test.go index 6910d2c..6149a3d 100644 --- a/run_test.go +++ b/run_test.go @@ -54,7 +54,7 @@ func TestPrintsNoStepDefinitionsIfNoneFound(t *testing.T) { } } -func TestShouldNotFailWhenHasPendingSteps(t *testing.T) { +func TestFailsOrPassesBasedOnStrictModeWhenHasPendingSteps(t *testing.T) { feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature)) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -72,6 +72,36 @@ func TestShouldNotFailWhenHasPendingSteps(t *testing.T) { if r.run() { t.Fatal("the suite should have passed") } + + r.strict = true + if !r.run() { + t.Fatal("the suite should have failed") + } +} + +func TestFailsOrPassesBasedOnStrictModeWhenHasUndefinedSteps(t *testing.T) { + feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + r := runner{ + fmt: progressFunc("progress", ioutil.Discard), + features: []*feature{&feature{Feature: feat}}, + initializer: func(s *Suite) { + s.Step(`^one$`, func() error { return nil }) + // two - is undefined + }, + } + + if r.run() { + t.Fatal("the suite should have passed") + } + + r.strict = true + if !r.run() { + t.Fatal("the suite should have failed") + } } func TestShouldFailOnError(t *testing.T) { diff --git a/suite.go b/suite.go index b256277..a3769ae 100644 --- a/suite.go +++ b/suite.go @@ -51,6 +51,7 @@ type Suite struct { failed bool randomSeed int64 stopOnFailure bool + strict bool // suite event handlers beforeSuiteHandlers []func() @@ -409,7 +410,7 @@ func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou f(outline, err) } - if err != nil && err != ErrUndefined && err != ErrPending { + if s.shouldFail(err) { failErr = err if s.stopOnFailure { return @@ -420,6 +421,18 @@ func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou return } +func (s *Suite) shouldFail(err error) bool { + if err == nil { + return false + } + + if err == ErrUndefined || err == ErrPending { + return s.strict + } + + return true +} + func (s *Suite) runFeature(f *feature) { s.fmt.Feature(f.Feature, f.Path, f.Content) @@ -447,7 +460,7 @@ func (s *Suite) runFeature(f *feature) { case *gherkin.Scenario: err = s.runScenario(t, f.Background) } - if err != nil && err != ErrUndefined && err != ErrPending { + if s.shouldFail(err) { s.failed = true if s.stopOnFailure { return