adds --strict flag to fail suite when there are pending or undefined steps

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change LOG
 
+**2017-05-04**
+- added **--strict** option in order to fail suite when there are pending
+  or undefined steps. By default, suite passes and treats pending or
+  undefined steps as TODOs.
+
 **2017-04-29** - **v0.7.0**
 - added support for nested steps. From now on, it is possible to return
   **godog.Steps** instead of an **error** in the step definition func.

flags.go

@@ -52,6 +52,7 @@ func FlagSet(opt *Options) *flag.FlagSet {
 	set.BoolVar(&opt.ShowStepDefinitions, "definitions", false, "Print all available step definitions.")
 	set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.")
 	set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.")
+	set.BoolVar(&opt.Strict, "strict", false, "Fail suite when there are pending or undefined steps.")
 	set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.")
 	set.Var(&randomSeed{&opt.Randomize}, "random", descRandomOption)
 	set.Usage = usage(set, opt.Output)

options.go

@@ -35,6 +35,9 @@ type Options struct {
 	// Stops on the first failure
 	StopOnFailure bool
 
+	// Fail suite when there are pending or undefined steps
+	Strict bool
+
 	// Forces ansi color stripping
 	NoColors bool
 

run.go

@@ -12,11 +12,11 @@ import (
 type initializer func(*Suite)
 
 type runner struct {
-	randomSeed    int64
-	stopOnFailure bool
-	features      []*feature
-	fmt           Formatter
-	initializer   initializer
+	randomSeed            int64
+	stopOnFailure, strict bool
+	features              []*feature
+	fmt                   Formatter
+	initializer           initializer
 }
 
 func (r *runner) concurrent(rate int) (failed bool) {
@@ -34,6 +34,7 @@ func (r *runner) concurrent(rate int) (failed bool) {
 				fmt:           r.fmt,
 				randomSeed:    r.randomSeed,
 				stopOnFailure: r.stopOnFailure,
+				strict:        r.strict,
 				features:      []*feature{feat},
 			}
 			r.initializer(suite)
@@ -59,6 +60,7 @@ func (r *runner) run() bool {
 		fmt:           r.fmt,
 		randomSeed:    r.randomSeed,
 		stopOnFailure: r.stopOnFailure,
+		strict:        r.strict,
 		features:      r.features,
 	}
 	r.initializer(suite)
@@ -116,6 +118,7 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt
 		features:      features,
 		randomSeed:    opt.Randomize,
 		stopOnFailure: opt.StopOnFailure,
+		strict:        opt.Strict,
 	}
 
 	// store chosen seed in environment, so it could be seen in formatter summary report

run_test.go

@@ -54,7 +54,7 @@ func TestPrintsNoStepDefinitionsIfNoneFound(t *testing.T) {
 	}
 }
 
-func TestShouldNotFailWhenHasPendingSteps(t *testing.T) {
+func TestFailsOrPassesBasedOnStrictModeWhenHasPendingSteps(t *testing.T) {
 	feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature))
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -72,6 +72,36 @@ func TestShouldNotFailWhenHasPendingSteps(t *testing.T) {
 	if r.run() {
 		t.Fatal("the suite should have passed")
 	}
+
+	r.strict = true
+	if !r.run() {
+		t.Fatal("the suite should have failed")
+	}
+}
+
+func TestFailsOrPassesBasedOnStrictModeWhenHasUndefinedSteps(t *testing.T) {
+	feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	r := runner{
+		fmt:      progressFunc("progress", ioutil.Discard),
+		features: []*feature{&feature{Feature: feat}},
+		initializer: func(s *Suite) {
+			s.Step(`^one$`, func() error { return nil })
+			// two - is undefined
+		},
+	}
+
+	if r.run() {
+		t.Fatal("the suite should have passed")
+	}
+
+	r.strict = true
+	if !r.run() {
+		t.Fatal("the suite should have failed")
+	}
 }
 
 func TestShouldFailOnError(t *testing.T) {

suite.go

@@ -51,6 +51,7 @@ type Suite struct {
 	failed        bool
 	randomSeed    int64
 	stopOnFailure bool
+	strict        bool
 
 	// suite event handlers
 	beforeSuiteHandlers    []func()
@@ -409,7 +410,7 @@ func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou
 				f(outline, err)
 			}
 
-			if err != nil && err != ErrUndefined && err != ErrPending {
+			if s.shouldFail(err) {
 				failErr = err
 				if s.stopOnFailure {
 					return
@@ -420,6 +421,18 @@ func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou
 	return
 }
 
+func (s *Suite) shouldFail(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if err == ErrUndefined || err == ErrPending {
+		return s.strict
+	}
+
+	return true
+}
+
 func (s *Suite) runFeature(f *feature) {
 	s.fmt.Feature(f.Feature, f.Path, f.Content)
 
@@ -447,7 +460,7 @@ func (s *Suite) runFeature(f *feature) {
 		case *gherkin.Scenario:
 			err = s.runScenario(t, f.Background)
 		}
-		if err != nil && err != ErrUndefined && err != ErrPending {
+		if s.shouldFail(err) {
 			s.failed = true
 			if s.stopOnFailure {
 				return