improve scenario outline tests and behavior

Makefile

@@ -2,8 +2,8 @@
 
 # runs all necessary tests
 test:
-	@sh -c 'if [ ! -z "$(go fmt ./...)" ]; then exit 1; fi'
+	@sh -c 'if [ ! -z "$(go fmt ./...)" ]; then exit 1; else echo "go fmt OK"; fi'
-	golint ./...
+	@sh -c 'if [ ! -z "$(golint ./...)" ]; then exit 1; else echo "golint OK"; fi'
 	go vet ./...
 	go test ./...
 	go run cmd/godog/main.go -f progress

features/background.feature

@@ -0,0 +1,75 @@
+Feature: run background
+  In order to test application behavior
+  As a test suite
+  I need to be able to run background correctly
+
+  Scenario: should run background steps
+    Given a feature "normal.feature" file:
+      """
+      Feature: with background
+
+        Background:
+          Given a feature path "features/load.feature:6"
+
+        Scenario: parse a scenario
+          When I parse features
+          Then I should have 1 scenario registered
+      """
+    When I run feature suite
+    Then the suite should have passed
+    And the following steps should be passed:
+      """
+      a feature path "features/load.feature:6"
+      I parse features
+      I should have 1 scenario registered
+      """
+
+  Scenario: should skip all consequent steps on failure
+    Given a feature "normal.feature" file:
+      """
+      Feature: with background
+
+        Background:
+          Given a failing step
+          And a feature path "features/load.feature:6"
+
+        Scenario: parse a scenario
+          When I parse features
+          Then I should have 1 scenario registered
+      """
+    When I run feature suite
+    Then the suite should have failed
+    And the following steps should be failed:
+      """
+      a failing step
+      """
+    And the following steps should be skipped:
+      """
+      a feature path "features/load.feature:6"
+      I parse features
+      I should have 1 scenario registered
+      """
+
+  Scenario: should continue undefined steps
+    Given a feature "normal.feature" file:
+      """
+      Feature: with background
+
+        Background:
+          Given an undefined step
+
+        Scenario: parse a scenario
+          When I do undefined action
+          Then I should have 1 scenario registered
+      """
+    When I run feature suite
+    Then the suite should have passed
+    And the following steps should be undefined:
+      """
+      an undefined step
+      I do undefined action
+      """
+    And the following steps should be skipped:
+      """
+      I should have 1 scenario registered
+      """

features/lang.feature

@@ -8,10 +8,12 @@ Savybė: užkrauti savybes
   Scenarijus: savybių užkrovimas iš aplanko
     Duota savybių aplankas "features"
     Kai aš išskaitau savybes
-    Tada aš turėčiau turėti 4 savybių failus:
+    Tada aš turėčiau turėti 6 savybių failus:
       """
+      features/background.feature
       features/events.feature
       features/lang.feature
       features/load.feature
+      features/outline.feature
       features/run.feature
       """

features/load.feature

@@ -6,11 +6,13 @@ Feature: load features
   Scenario: load features within path
     Given a feature path "features"
     When I parse features
-    Then I should have 4 feature files:
+    Then I should have 6 feature files:
       """
+      features/background.feature
       features/events.feature
       features/lang.feature
       features/load.feature
+      features/outline.feature
       features/run.feature
       """
 

features/outline.feature

@@ -0,0 +1,100 @@
+Feature: run outline
+  In order to test application behavior
+  As a test suite
+  I need to be able to run outline scenarios
+
+  Scenario: should run a normal outline
+    Given a feature "normal.feature" file:
+      """
+      Feature: outline
+
+        Background:
+          Given passing step
+
+        Scenario Outline: parse a scenario
+          Given a feature path "<path>"
+          When I parse features
+          Then I should have <num> scenario registered
+
+          Examples:
+            | path                    | num |
+            | features/load.feature:6 | 1   |
+            | features/load.feature:3 | 0   |
+      """
+    When I run feature suite
+    Then the suite should have passed
+    And the following steps should be passed:
+      """
+      a passing step
+      I parse features
+      a feature path "features/load.feature:6"
+      a feature path "features/load.feature:3"
+      I should have 1 scenario registered
+      I should have 0 scenario registered
+      """
+
+  Scenario: should continue through examples on failure
+    Given a feature "normal.feature" file:
+      """
+      Feature: outline
+
+        Background:
+          Given passing step
+
+        Scenario Outline: parse a scenario
+          Given a feature path "<path>"
+          When I parse features
+          Then I should have <num> scenario registered
+
+          Examples:
+            | path                    | num |
+            | features/load.feature:6 | 5   |
+            | features/load.feature:3 | 0   |
+      """
+    When I run feature suite
+    Then the suite should have failed
+    And the following steps should be passed:
+      """
+      a passing step
+      I parse features
+      a feature path "features/load.feature:6"
+      a feature path "features/load.feature:3"
+      I should have 0 scenario registered
+      """
+    And the following steps should be failed:
+      """
+      I should have 5 scenario registered
+      """
+
+  Scenario: should skip examples on background failure
+    Given a feature "normal.feature" file:
+      """
+      Feature: outline
+
+        Background:
+          Given a failing step
+
+        Scenario Outline: parse a scenario
+          Given a feature path "<path>"
+          When I parse features
+          Then I should have <num> scenario registered
+
+          Examples:
+            | path                    | num |
+            | features/load.feature:6 | 1   |
+            | features/load.feature:3 | 0   |
+      """
+    When I run feature suite
+    Then the suite should have failed
+    And the following steps should be skipped:
+      """
+      I parse features
+      a feature path "features/load.feature:6"
+      a feature path "features/load.feature:3"
+      I should have 0 scenario registered
+      I should have 1 scenario registered
+      """
+    And the following steps should be failed:
+      """
+      a failing step
+      """

fmt.go

@@ -77,9 +77,31 @@ type Formatter interface {
 	Summary()
 }
 
-// failed represents a failed step data structure
+type stepType int
-// with all necessary references
+
-type failed struct {
+const (
+	passed stepType = iota
+	failed
+	skipped
+	undefined
+	pending
+)
+
+func (st stepType) clr() color {
+	switch st {
+	case passed:
+		return green
+	case failed:
+		return red
+	case skipped:
+		return cyan
+	default:
+		return yellow
+	}
+}
+
+type stepResult struct {
+	typ     stepType
 	feature *feature
 	owner   interface{}
 	step    *gherkin.Step
@@ -87,55 +109,21 @@ type failed struct {
 	err     error
 }
 
-func (f failed) line() string {
+func (f stepResult) line() string {
 	return fmt.Sprintf("%s:%d", f.feature.Path, f.step.Location.Line)
 }
 
-// passed represents a successful step data structure
-// with all necessary references
-type passed struct {
-	feature *feature
-	owner   interface{}
-	step    *gherkin.Step
-	def     *StepDef
-}
-
-// skipped represents a skipped step data structure
-// with all necessary references
-type skipped struct {
-	feature *feature
-	owner   interface{}
-	step    *gherkin.Step
-}
-
-// undefined represents an undefined step data structure
-// with all necessary references
-type undefined struct {
-	feature *feature
-	owner   interface{}
-	step    *gherkin.Step
-}
-
-// pending represents a pending step data structure
-// with all necessary references
-type pending struct {
-	feature *feature
-	owner   interface{}
-	step    *gherkin.Step
-	def     *StepDef
-}
-
 type basefmt struct {
 	owner  interface{}
 	indent int
 
 	started   time.Time
 	features  []*feature
-	failed    []*failed
+	failed    []*stepResult
-	passed    []*passed
+	passed    []*stepResult
-	skipped   []*skipped
+	skipped   []*stepResult
-	undefined []*undefined
+	undefined []*stepResult
-	pending   []*pending
+	pending   []*stepResult
 }
 
 func (f *basefmt) Node(n interface{}) {
@@ -154,27 +142,56 @@ func (f *basefmt) Feature(ft *gherkin.Feature, p string) {
 }
 
 func (f *basefmt) Passed(step *gherkin.Step, match *StepDef) {
-	s := &passed{owner: f.owner, feature: f.features[len(f.features)-1], step: step, def: match}
+	s := &stepResult{
+		owner:   f.owner,
+		feature: f.features[len(f.features)-1],
+		step:    step,
+		def:     match,
+		typ:     passed,
+	}
 	f.passed = append(f.passed, s)
 }
 
 func (f *basefmt) Skipped(step *gherkin.Step) {
-	s := &skipped{owner: f.owner, feature: f.features[len(f.features)-1], step: step}
+	s := &stepResult{
+		owner:   f.owner,
+		feature: f.features[len(f.features)-1],
+		step:    step,
+		typ:     skipped,
+	}
 	f.skipped = append(f.skipped, s)
 }
 
 func (f *basefmt) Undefined(step *gherkin.Step) {
-	s := &undefined{owner: f.owner, feature: f.features[len(f.features)-1], step: step}
+	s := &stepResult{
+		owner:   f.owner,
+		feature: f.features[len(f.features)-1],
+		step:    step,
+		typ:     undefined,
+	}
 	f.undefined = append(f.undefined, s)
 }
 
 func (f *basefmt) Failed(step *gherkin.Step, match *StepDef, err error) {
-	s := &failed{owner: f.owner, feature: f.features[len(f.features)-1], step: step, def: match, err: err}
+	s := &stepResult{
+		owner:   f.owner,
+		feature: f.features[len(f.features)-1],
+		step:    step,
+		def:     match,
+		err:     err,
+		typ:     failed,
+	}
 	f.failed = append(f.failed, s)
 }
 
 func (f *basefmt) Pending(step *gherkin.Step, match *StepDef) {
-	s := &pending{owner: f.owner, feature: f.features[len(f.features)-1], step: step, def: match}
+	s := &stepResult{
+		owner:   f.owner,
+		feature: f.features[len(f.features)-1],
+		step:    step,
+		def:     match,
+		typ:     pending,
+	}
 	f.pending = append(f.pending, s)
 }
 

fmt_pretty.go

@@ -28,7 +28,7 @@ type pretty struct {
 	backgroundSteps int
 
 	// outline
-	outlineSteps       []interface{}
+	outlineSteps       []*stepResult
 	outlineNumExample  int
 	outlineNumExamples int
 }
@@ -96,7 +96,7 @@ func (f *pretty) Node(node interface{}) {
 // Summary sumarize the feature formatter output
 func (f *pretty) Summary() {
 	// failed steps on background are not scenarios
-	var failedScenarios []*failed
+	var failedScenarios []*stepResult
 	for _, fail := range f.failed {
 		switch fail.owner.(type) {
 		case *gherkin.Scenario:
@@ -129,31 +129,29 @@ func (f *pretty) Summary() {
 }
 
 func (f *pretty) printOutlineExample(outline *gherkin.ScenarioOutline) {
-	var failed error
+	var msg string
 	clr := green
 
 	example := outline.Examples[f.outlineNumExample]
 	firstExample := f.outlineNumExamples == len(example.TableBody)
 	printSteps := firstExample && f.outlineNumExample == 0
 
-	// var replace make(map[])
+	for i, res := range f.outlineSteps {
-	for i, act := range f.outlineSteps {
-		_, _, def, c, err := f.stepDetails(act)
 		// determine example row status
 		switch {
-		case err != nil:
+		case res.typ == failed:
-			failed = err
+			msg = res.err.Error()
-			clr = red
+			clr = res.typ.clr()
-		case c == yellow:
+		case res.typ == undefined || res.typ == pending:
-			clr = yellow
+			clr = res.typ.clr()
-		case c == cyan && clr == green:
+		case res.typ == skipped && clr == green:
 			clr = cyan
 		}
 		if printSteps {
 			// in first example, we need to print steps
 			var text string
 			ostep := outline.Steps[i]
-			if def != nil {
+			if res.def != nil {
 				if m := outlinePlaceholderRegexp.FindAllStringIndex(ostep.Text, -1); len(m) > 0 {
 					var pos int
 					for i := 0; i < len(m); i++ {
@@ -166,7 +164,7 @@ func (f *pretty) printOutlineExample(outline *gherkin.ScenarioOutline) {
 				} else {
 					text = cl(ostep.Text, cyan)
 				}
-				text += s(f.commentPos-f.length(ostep)+1) + cl(fmt.Sprintf("# %s", def.funcName()), black)
+				text += s(f.commentPos-f.length(ostep)+1) + cl(fmt.Sprintf("# %s", res.def.funcName()), black)
 			} else {
 				text = cl(ostep.Text, cyan)
 			}
@@ -196,8 +194,8 @@ func (f *pretty) printOutlineExample(outline *gherkin.ScenarioOutline) {
 	fmt.Println(s(f.indent*3) + "| " + strings.Join(cells, " | ") + " |")
 
 	// if there is an error
-	if failed != nil {
+	if msg != "" {
-		fmt.Println(s(f.indent*3) + bcl(failed, red))
+		fmt.Println(s(f.indent*4) + bcl(msg, red))
 	}
 }
 
@@ -241,43 +239,9 @@ func (f *pretty) printStep(step *gherkin.Step, def *StepDef, c color) {
 	}
 }
 
-func (f *pretty) stepDetails(stepAction interface{}) (owner interface{}, step *gherkin.Step, def *StepDef, c color, err error) {
+func (f *pretty) printStepKind(res *stepResult) {
-	switch typ := stepAction.(type) {
-	case *passed:
-		step = typ.step
-		def = typ.def
-		owner = typ.owner
-		c = green
-	case *failed:
-		step = typ.step
-		def = typ.def
-		owner = typ.owner
-		err = typ.err
-		c = red
-	case *skipped:
-		step = typ.step
-		owner = typ.owner
-		c = cyan
-	case *undefined:
-		step = typ.step
-		owner = typ.owner
-		c = yellow
-	case *pending:
-		step = typ.step
-		def = typ.def
-		owner = typ.owner
-		c = yellow
-	default:
-		fatal(fmt.Errorf("unexpected step type received: %T", typ))
-	}
-	return
-}
-
-func (f *pretty) printStepKind(stepAction interface{}) {
-	owner, step, def, c, err := f.stepDetails(stepAction)
-
 	// do not print background more than once
-	if _, ok := owner.(*gherkin.Background); ok {
+	if _, ok := res.owner.(*gherkin.Background); ok {
 		switch {
 		case f.backgroundSteps == 0:
 			return
@@ -286,22 +250,22 @@ func (f *pretty) printStepKind(stepAction interface{}) {
 		}
 	}
 
-	if outline, ok := owner.(*gherkin.ScenarioOutline); ok {
+	if outline, ok := res.owner.(*gherkin.ScenarioOutline); ok {
-		f.outlineSteps = append(f.outlineSteps, stepAction)
+		f.outlineSteps = append(f.outlineSteps, res)
 		if len(f.outlineSteps) == len(outline.Steps) {
 			// an outline example steps has went through
 			f.printOutlineExample(outline)
-			f.outlineSteps = []interface{}{}
+			f.outlineSteps = []*stepResult{}
 			f.outlineNumExamples--
 		}
 		return
 	}
 
-	f.printStep(step, def, c)
+	f.printStep(res.step, res.def, res.typ.clr())
-	if err != nil {
+	if res.err != nil {
-		fmt.Println(s(f.indent*2) + bcl(err, red))
+		fmt.Println(s(f.indent*2) + bcl(res.err, red))
 	}
-	if _, ok := stepAction.(*pending); ok {
+	if res.typ == pending {
 		fmt.Println(s(f.indent*3) + cl("TODO: write pending definition", yellow))
 	}
 }

fmt_progress.go

@@ -45,17 +45,17 @@ func (f *progress) Summary() {
 	f.basefmt.Summary()
 }
 
-func (f *progress) step(step interface{}) {
+func (f *progress) step(res *stepResult) {
-	switch step.(type) {
+	switch res.typ {
-	case *passed:
+	case passed:
 		fmt.Print(cl(".", green))
-	case *skipped:
+	case skipped:
 		fmt.Print(cl("-", cyan))
-	case *failed:
+	case failed:
 		fmt.Print(cl("F", red))
-	case *undefined:
+	case undefined:
 		fmt.Print(cl("U", yellow))
-	case *pending:
+	case pending:
 		fmt.Print(cl("P", yellow))
 	}
 	f.steps++

suite.go

@@ -298,6 +298,11 @@ func (s *suite) runStep(step *gherkin.Step, prevStepErr error) (err error) {
 		return nil
 	}
 
+	// run before step handlers
+	for _, f := range s.beforeStepHandlers {
+		f(step)
+	}
+
 	defer func() {
 		if e := recover(); e != nil {
 			err, ok := e.(error)
@@ -317,16 +322,17 @@ func (s *suite) runStep(step *gherkin.Step, prevStepErr error) (err error) {
 	default:
 		s.fmt.Failed(step, match, err)
 	}
+
+	// run after step handlers
+	for _, f := range s.afterStepHandlers {
+		f(step, err)
+	}
 	return
 }
 
-func (s *suite) runSteps(steps []*gherkin.Step) (err error) {
+func (s *suite) runSteps(steps []*gherkin.Step, prevErr error) (err error) {
+	err = prevErr
 	for _, step := range steps {
-		// run before step handlers
-		for _, f := range s.beforeStepHandlers {
-			f(step)
-		}
-
 		stepErr := s.runStep(step, err)
 		switch stepErr {
 		case ErrUndefined:
@@ -337,11 +343,6 @@ func (s *suite) runSteps(steps []*gherkin.Step) (err error) {
 		default:
 			err = stepErr
 		}
-
-		// run after step handlers
-		for _, f := range s.afterStepHandlers {
-			f(step, err)
-		}
 	}
 	return
 }
@@ -352,7 +353,7 @@ func (s *suite) skipSteps(steps []*gherkin.Step) {
 	}
 }
 
-func (s *suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Background) (err error) {
+func (s *suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Background) (failErr error) {
 	s.fmt.Node(outline)
 
 	for _, example := range outline.Examples {
@@ -381,24 +382,21 @@ func (s *suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou
 				steps = append(steps, step)
 			}
 			// run background
+			var err error
 			if b != nil {
-				err = s.runSteps(b.Steps)
+				err = s.runSteps(b.Steps, err)
-			}
-			switch err {
-			case ErrUndefined:
-				s.skipSteps(steps)
-			case nil:
-				err = s.runSteps(steps)
-			default:
-				s.skipSteps(steps)
 			}
+			err = s.runSteps(steps, err)
 
 			for _, f := range s.afterScenarioHandlers {
 				f(outline, err)
 			}
 
-			if s.stopOnFailure && err != ErrUndefined {
+			if err != nil && err != ErrUndefined && err != ErrPending {
-				return
+				failErr = err
+				if s.stopOnFailure {
+					return
+				}
 			}
 		}
 	}
@@ -435,21 +433,12 @@ func (s *suite) runScenario(scenario *gherkin.Scenario, b *gherkin.Background) (
 
 	// background
 	if b != nil {
-		err = s.runSteps(b.Steps)
+		err = s.runSteps(b.Steps, err)
 	}
 
 	// scenario
 	s.fmt.Node(scenario)
-	switch err {
+	err = s.runSteps(scenario.Steps, err)
-	case ErrPending:
-		s.skipSteps(scenario.Steps)
-	case ErrUndefined:
-		s.skipSteps(scenario.Steps)
-	case nil:
-		err = s.runSteps(scenario.Steps)
-	default:
-		s.skipSteps(scenario.Steps)
-	}
 
 	// run after scenario handlers
 	for _, f := range s.afterScenarioHandlers {

suite_test.go

@@ -38,6 +38,9 @@ func SuiteContext(s Suite) {
 	s.Step(`^pending step$`, func() error {
 		return ErrPending
 	})
+	s.Step(`^passing step$`, func() error {
+		return nil
+	})
 }
 
 type firedEvent struct {
@@ -63,8 +66,7 @@ func (s *suiteContext) ResetBeforeEachScenario(interface{}) {
 
 func (s *suiteContext) followingStepsShouldHave(status string, steps *gherkin.DocString) error {
 	var expected = strings.Split(steps.Content, "\n")
-	var actual, unmatched []string
+	var actual, unmatched, matched []string
-	var matched []int
 
 	switch status {
 	case "passed":
@@ -96,23 +98,23 @@ func (s *suiteContext) followingStepsShouldHave(status string, steps *gherkin.Do
 	}
 
 	for _, a := range actual {
-		for i, e := range expected {
+		for _, e := range expected {
 			if a == e {
-				matched = append(matched, i)
+				matched = append(matched, e)
 				break
 			}
 		}
 	}
 
-	if len(matched) == len(expected) {
+	if len(matched) >= len(expected) {
 		return nil
 	}
-
+	for _, s := range expected {
-	for i, s := range expected {
 		var found bool
 		for _, m := range matched {
-			if i == m {
+			if s == m {
 				found = true
+				break
 			}
 		}
 		if !found {
@@ -120,7 +122,7 @@ func (s *suiteContext) followingStepsShouldHave(status string, steps *gherkin.Do
 		}
 	}
 
-	return fmt.Errorf("the steps: %s - is not %s", strings.Join(unmatched, ", "), status)
+	return fmt.Errorf("the steps: %s - are not %s", strings.Join(unmatched, ", "), status)
 }
 
 func (s *suiteContext) iAmListeningToSuiteEvents() error {