adds more tests for multistep execution

features/formatter/events.feature

@@ -14,7 +14,7 @@ Feature: event stream formatter
       """
 
   Scenario: should process simple scenario
-    Given a feature path "features/load.feature:22"
+    Given a feature path "features/load.feature:23"
     When I run feature suite with formatter "events"
     Then the following events should be fired:
       """
@@ -35,7 +35,7 @@ Feature: event stream formatter
       """
 
   Scenario: should process outline scenario
-    Given a feature path "features/load.feature:30"
+    Given a feature path "features/load.feature:31"
     When I run feature suite with formatter "events"
     Then the following events should be fired:
       """

fmt_progress_test.go

@@ -34,7 +34,6 @@ func TestProgressFormatterOutput(t *testing.T) {
 		},
 	}
 
-	// var zeroDuration time.Duration
 	expected := `
 ...F-.P-.UU.....F..P..U 23
 
@@ -88,3 +87,113 @@ func trimAllLines(s string) string {
 	}
 	return strings.Join(lines, "\n")
 }
+
+var basicGherkinFeature = `
+Feature: basic
+
+  Scenario: passing scenario
+	When step1
+	Then step2
+`
+
+func TestProgressFormatterWhenStepPanics(t *testing.T) {
+	feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	w := colors.Uncolored(&buf)
+	r := runner{
+		fmt:      progressFunc("progress", w),
+		features: []*feature{&feature{Feature: feat}},
+		initializer: func(s *Suite) {
+			s.Step(`^step1$`, func() error { return nil })
+			s.Step(`^step2$`, func() error { panic("omg") })
+		},
+	}
+
+	if !r.run() {
+		t.Fatal("the suite should have failed")
+	}
+
+	out := buf.String()
+	if idx := strings.Index(out, "github.com/DATA-DOG/godog/fmt_progress_test.go:116"); idx == -1 {
+		t.Fatal("expected to find panic stacktrace")
+	}
+}
+
+func TestProgressFormatterWithPassingMultisteps(t *testing.T) {
+	feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	w := colors.Uncolored(&buf)
+	r := runner{
+		fmt:      progressFunc("progress", w),
+		features: []*feature{&feature{Feature: feat}},
+		initializer: func(s *Suite) {
+			s.Step(`^sub1$`, func() error { return nil })
+			s.Step(`^sub-sub$`, func() error { return nil })
+			s.Step(`^sub2$`, func() Steps { return Steps{"sub-sub", "sub1", "step1"} })
+			s.Step(`^step1$`, func() error { return nil })
+			s.Step(`^step2$`, func() Steps { return Steps{"sub1", "sub2"} })
+		},
+	}
+
+	if r.run() {
+		t.Fatal("the suite should have passed")
+	}
+}
+
+func TestProgressFormatterWithFailingMultisteps(t *testing.T) {
+	feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	w := colors.Uncolored(&buf)
+	r := runner{
+		fmt:      progressFunc("progress", w),
+		features: []*feature{&feature{Feature: feat}},
+		initializer: func(s *Suite) {
+			s.Step(`^sub1$`, func() error { return nil })
+			s.Step(`^sub-sub$`, func() error { return fmt.Errorf("errored") })
+			s.Step(`^sub2$`, func() Steps { return Steps{"sub-sub", "sub1", "step1"} })
+			s.Step(`^step1$`, func() error { return nil })
+			s.Step(`^step2$`, func() Steps { return Steps{"sub1", "sub2"} })
+		},
+	}
+
+	if !r.run() {
+		t.Fatal("the suite should have failed")
+	}
+}
+
+func TestProgressFormatterWithPanicInMultistep(t *testing.T) {
+	feat, err := gherkin.ParseFeature(strings.NewReader(basicGherkinFeature))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var buf bytes.Buffer
+	w := colors.Uncolored(&buf)
+	r := runner{
+		fmt:      progressFunc("progress", w),
+		features: []*feature{&feature{Feature: feat}},
+		initializer: func(s *Suite) {
+			s.Step(`^sub1$`, func() error { return nil })
+			s.Step(`^sub-sub$`, func() error { return nil })
+			s.Step(`^sub2$`, func() []string { return []string{"sub-sub", "sub1", "step1"} })
+			s.Step(`^step1$`, func() error { return nil })
+			s.Step(`^step2$`, func() []string { return []string{"sub1", "sub2"} })
+		},
+	}
+
+	if !r.run() {
+		t.Fatal("the suite should have failed")
+	}
+}

run.go

@@ -54,7 +54,7 @@ func (r *runner) concurrent(rate int) (failed bool) {
 	return
 }
 
-func (r *runner) run() (failed bool) {
+func (r *runner) run() bool {
 	suite := &Suite{
 		fmt:           r.fmt,
 		randomSeed:    r.randomSeed,

suite.go

@@ -211,18 +211,7 @@ func (s *Suite) runStep(step *gherkin.Step, prevStepErr error) (err error) {
 	match := s.matchStep(step)
 	s.fmt.Defined(step, match)
 
-	// @TODO custom undefined err here to pass step text for snippet
+	// user multistep definitions may panic
-	// @TODO user multistep definitions may panic
-	if s.maybeUndefined(match) {
-		s.fmt.Undefined(step)
-		return ErrUndefined
-	}
-
-	if prevStepErr != nil {
-		s.fmt.Skipped(step)
-		return nil
-	}
-
 	defer func() {
 		if e := recover(); e != nil {
 			err = &traceError{
@@ -230,6 +219,15 @@ func (s *Suite) runStep(step *gherkin.Step, prevStepErr error) (err error) {
 				stack: callStack(),
 			}
 		}
+
+		if prevStepErr != nil {
+			return
+		}
+
+		if err == ErrUndefined {
+			return
+		}
+
 		switch err {
 		case nil:
 			s.fmt.Passed(step, match)
@@ -245,6 +243,17 @@ func (s *Suite) runStep(step *gherkin.Step, prevStepErr error) (err error) {
 		}
 	}()
 
+	// @TODO custom undefined err here to pass step text for snippet
+	if s.maybeUndefined(match) {
+		s.fmt.Undefined(step)
+		return ErrUndefined
+	}
+
+	if prevStepErr != nil {
+		s.fmt.Skipped(step)
+		return nil
+	}
+
 	// run before step handlers
 	for _, f := range s.beforeStepHandlers {
 		f(step)
@@ -303,9 +312,15 @@ func (s *Suite) matchStepText(text string) *StepDef {
 				args = append(args, m)
 			}
 
-			// @TODO copy step def
+			// since we need to assign arguments
-			h.args = args
+			// better to copy the step definition
-			return h
+			return &StepDef{
+				args:    args,
+				hv:      h.hv,
+				Expr:    h.Expr,
+				Handler: h.Handler,
+				nested:  h.nested,
+			}
 		}
 	}
 	return nil

suite_test.go

@@ -20,21 +20,28 @@ func TestMain(m *testing.M) {
 	format := "progress" // non verbose mode
 	concurrency := 4
 
+	var specific bool
 	for _, arg := range os.Args[1:] {
 		if arg == "-test.v=true" { // go test transforms -v option - verbose mode
 			format = "pretty"
 			concurrency = 1
 			break
 		}
+		if strings.Index(arg, "-test.run") == 0 {
+			specific = true
+		}
+	}
+	var status int
+	if !specific {
+		status = RunWithOptions("godog", func(s *Suite) {
+			SuiteContext(s)
+		}, Options{
+			Format:      format, // pretty format for verbose mode, otherwise - progress
+			Paths:       []string{"features"},
+			Concurrency: concurrency,           // concurrency for verbose mode is 1
+			Randomize:   time.Now().UnixNano(), // randomize scenario execution order
+		})
 	}
-	status := RunWithOptions("godog", func(s *Suite) {
-		SuiteContext(s)
-	}, Options{
-		Format:      format, // pretty format for verbose mode, otherwise - progress
-		Paths:       []string{"features"},
-		Concurrency: concurrency,           // concurrency for verbose mode is 1
-		Randomize:   time.Now().UnixNano(), // randomize scenario execution order
-	})
 
 	if st := m.Run(); st > status {
 		status = st