exit status on failure, test godog with godog on travis

.travis.yml

@@ -6,7 +6,12 @@ go:
   - tip
 
 script:
+  # pull all dependencies
   - go get -t ./...
 
+  # run standard go tests
   - go test -v ./...
   - go test -race ./...
+
+  # test me with myself
+  - go run cmd/godog/main.go

cmd/godog/main.go

@@ -16,19 +16,21 @@ func main() {
 	stdout := ansicolor.NewAnsiColorWriter(os.Stdout)
 
 	builtFile := fmt.Sprintf("%s/%dgodog.go", os.TempDir(), time.Now().UnixNano())
-	defer os.Remove(builtFile) // comment out for debug
 
 	buf, err := godog.Build()
 	if err != nil {
+		os.Remove(builtFile)
 		panic(err)
 	}
 
 	w, err := os.Create(builtFile)
 	if err != nil {
+		os.Remove(builtFile)
 		panic(err)
 	}
 	_, err = w.Write(buf)
 	if err != nil {
+		os.Remove(builtFile)
 		panic(err)
 	}
 	w.Close()
@@ -43,8 +45,10 @@ func main() {
 	err = cmd.Run()
 	switch err.(type) {
 	case *exec.ExitError:
+		os.Remove(builtFile)
 		os.Exit(1)
 	case *exec.Error:
+		os.Remove(builtFile)
 		panic(err)
 	}
 }

suite.go

@@ -3,6 +3,7 @@ package godog
 import (
 	"flag"
 	"fmt"
+	"os"
 	"reflect"
 	"regexp"
 	"runtime"
@@ -10,6 +11,14 @@ import (
 	"github.com/DATA-DOG/godog/gherkin"
 )
 
+type stepsStatus int
+
+const (
+	stepsStatusPassed stepsStatus = iota
+	stepsStatusFailed
+	stepsStatusUndefined
+)
+
 type BeforeScenarioHandler interface {
 	BeforeScenario(scenario *gherkin.Scenario)
 }
@@ -66,7 +75,7 @@ type suite struct {
 	features               []*gherkin.Feature
 	fmt                    Formatter
 
-	stop bool
+	failed bool
 }
 
 // New initializes a suite which supports the Suite
@@ -122,12 +131,15 @@ func (s *suite) Run() {
 
 	for _, f := range s.features {
 		s.runFeature(f)
-		if s.stop {
+		if s.failed && cfg.stopOnFailure {
 			// stop on first failure
 			break
 		}
 	}
 	s.fmt.Summary()
+	if s.failed {
+		os.Exit(1)
+	}
 }
 
 func (s *suite) runStep(step *gherkin.Step) (err error) {
@@ -168,18 +180,21 @@ func (s *suite) runStep(step *gherkin.Step) (err error) {
 	return
 }
 
-func (s *suite) runSteps(steps []*gherkin.Step) bool {
+func (s *suite) runSteps(steps []*gherkin.Step) (st stepsStatus) {
-	var failed bool
 	for _, step := range steps {
-		if failed {
+		if st != stepsStatusPassed {
 			s.fmt.Skipped(step)
 			continue
 		}
-		if err := s.runStep(step); err != nil {
+		err := s.runStep(step)
-			failed = true
+		switch {
+		case err == errPending:
+			st = stepsStatusUndefined
+		case err != nil:
+			st = stepsStatusFailed
 		}
 	}
-	return failed
+	return
 }
 
 func (s *suite) skipSteps(steps []*gherkin.Step) {
@@ -191,7 +206,7 @@ func (s *suite) skipSteps(steps []*gherkin.Step) {
 func (s *suite) runFeature(f *gherkin.Feature) {
 	s.fmt.Node(f)
 	for _, scenario := range f.Scenarios {
-		var failed bool
+		var status stepsStatus
 
 		// run before scenario handlers
 		for _, h := range s.beforeScenarioHandlers {
@@ -201,23 +216,28 @@ func (s *suite) runFeature(f *gherkin.Feature) {
 		// background
 		if f.Background != nil {
 			s.fmt.Node(f.Background)
-			failed = s.runSteps(f.Background.Steps)
+			status = s.runSteps(f.Background.Steps)
 		}
 
 		// scenario
 		s.fmt.Node(scenario)
-		if failed {
+		switch {
+		case status == stepsStatusFailed:
 			s.skipSteps(scenario.Steps)
-		} else {
+		case status == stepsStatusUndefined:
-			failed = s.runSteps(scenario.Steps)
+			s.skipSteps(scenario.Steps)
+		default:
+			status = s.runSteps(scenario.Steps)
 		}
 
-		if failed && cfg.stopOnFailure {
+		if status == stepsStatusFailed {
-			s.stop = true
+			s.failed = true
+			if cfg.stopOnFailure {
 				return
 			}
 		}
 	}
+}
 
 func (st *suite) printStepDefinitions() {
 	var longest int

suite_test.go

@@ -18,6 +18,8 @@ func (s *suiteFeature) BeforeScenario(scenario *gherkin.Scenario) {
 	cfg.paths = []string{}
 	// reset hook test references
 	s.befScenarioHook = nil
+	// reset formatter, which collects all details
+	s.fmt = &testFormatter{}
 }
 
 func (s *suiteFeature) iHaveBeforeScenarioHook(args ...*Arg) error {

utils_test.go

@@ -0,0 +1,40 @@
+package godog
+
+import "github.com/DATA-DOG/godog/gherkin"
+
+type testFormatter struct {
+	features  []*gherkin.Feature
+	scenarios []*gherkin.Scenario
+
+	failed    []*failed
+	passed    []*passed
+	skipped   []*skipped
+	undefined []*undefined
+}
+
+func (f *testFormatter) Node(node interface{}) {
+	switch t := node.(type) {
+	case *gherkin.Feature:
+		f.features = append(f.features, t)
+	case *gherkin.Scenario:
+		f.scenarios = append(f.scenarios, t)
+	}
+}
+
+func (f *testFormatter) Summary() {}
+
+func (f *testFormatter) Passed(step *gherkin.Step, match *stepMatchHandler) {
+	f.passed = append(f.passed, &passed{step})
+}
+
+func (f *testFormatter) Skipped(step *gherkin.Step) {
+	f.skipped = append(f.skipped, &skipped{step})
+}
+
+func (f *testFormatter) Undefined(step *gherkin.Step) {
+	f.undefined = append(f.undefined, &undefined{step})
+}
+
+func (f *testFormatter) Failed(step *gherkin.Step, match *stepMatchHandler, err error) {
+	f.failed = append(f.failed, &failed{step, err})
+}