Merge pull request #204 from lonnblad/feature/junit-concurrency

Refactored basfmt and junitFormatter to support concurrency

fmt.go

@@ -107,6 +107,14 @@ type Formatter interface {
 	Summary()
 }
 
+// ConcurrentFormatter is an interface for a Concurrent
+// version of the Formatter interface.
+type ConcurrentFormatter interface {
+	Formatter
+	Copy(ConcurrentFormatter)
+	Sync(ConcurrentFormatter)
+}
+
 // FormatterFunc builds a formatter with given
 // suite name and io.Writer to record output
 type FormatterFunc func(string, io.Writer) Formatter
@@ -156,6 +164,7 @@ type stepResult struct {
 	feature *feature
 	owner   interface{}
 	step    *gherkin.Step
+	time    time.Time
 	def     *StepDef
 	err     error
 }
@@ -213,6 +222,8 @@ func (f stepResult) scenarioLine() string {
 }
 
 type basefmt struct {
+	suiteName string
+
 	out    io.Writer
 	owner  interface{}
 	indent int
@@ -228,10 +239,28 @@ type basefmt struct {
 
 func (f *basefmt) Node(n interface{}) {
 	switch t := n.(type) {
-	case *gherkin.TableRow:
-		f.owner = t
 	case *gherkin.Scenario:
 		f.owner = t
+		feature := f.features[len(f.features)-1]
+		feature.Scenarios = append(feature.Scenarios, &scenario{Name: t.Name, time: timeNowFunc()})
+	case *gherkin.ScenarioOutline:
+		feature := f.features[len(f.features)-1]
+		feature.Scenarios = append(feature.Scenarios, &scenario{OutlineName: t.Name})
+	case *gherkin.TableRow:
+		f.owner = t
+
+		feature := f.features[len(f.features)-1]
+		lastExample := feature.Scenarios[len(feature.Scenarios)-1]
+
+		newExample := scenario{OutlineName: lastExample.OutlineName, ExampleNo: lastExample.ExampleNo + 1, time: timeNowFunc()}
+		newExample.Name = fmt.Sprintf("%s #%d", newExample.OutlineName, newExample.ExampleNo)
+
+		const firstExample = 1
+		if newExample.ExampleNo == firstExample {
+			feature.Scenarios[len(feature.Scenarios)-1] = &newExample
+		} else {
+			feature.Scenarios = append(feature.Scenarios, &newExample)
+		}
 	}
 }
 
@@ -240,7 +269,7 @@ func (f *basefmt) Defined(*gherkin.Step, *StepDef) {
 }
 
 func (f *basefmt) Feature(ft *gherkin.Feature, p string, c []byte) {
-	f.features = append(f.features, &feature{Path: p, Feature: ft})
+	f.features = append(f.features, &feature{Path: p, Feature: ft, time: timeNowFunc()})
 }
 
 func (f *basefmt) Passed(step *gherkin.Step, match *StepDef) {
@@ -250,8 +279,11 @@ func (f *basefmt) Passed(step *gherkin.Step, match *StepDef) {
 		step:    step,
 		def:     match,
 		typ:     passed,
+		time:    timeNowFunc(),
 	}
 	f.passed = append(f.passed, s)
+
+	f.features[len(f.features)-1].appendStepResult(s)
 }
 
 func (f *basefmt) Skipped(step *gherkin.Step, match *StepDef) {
@@ -261,8 +293,11 @@ func (f *basefmt) Skipped(step *gherkin.Step, match *StepDef) {
 		step:    step,
 		def:     match,
 		typ:     skipped,
+		time:    timeNowFunc(),
 	}
 	f.skipped = append(f.skipped, s)
+
+	f.features[len(f.features)-1].appendStepResult(s)
 }
 
 func (f *basefmt) Undefined(step *gherkin.Step, match *StepDef) {
@@ -272,8 +307,11 @@ func (f *basefmt) Undefined(step *gherkin.Step, match *StepDef) {
 		step:    step,
 		def:     match,
 		typ:     undefined,
+		time:    timeNowFunc(),
 	}
 	f.undefined = append(f.undefined, s)
+
+	f.features[len(f.features)-1].appendStepResult(s)
 }
 
 func (f *basefmt) Failed(step *gherkin.Step, match *StepDef, err error) {
@@ -284,8 +322,11 @@ func (f *basefmt) Failed(step *gherkin.Step, match *StepDef, err error) {
 		def:     match,
 		err:     err,
 		typ:     failed,
+		time:    timeNowFunc(),
 	}
 	f.failed = append(f.failed, s)
+
+	f.features[len(f.features)-1].appendStepResult(s)
 }
 
 func (f *basefmt) Pending(step *gherkin.Step, match *StepDef) {
@@ -295,8 +336,11 @@ func (f *basefmt) Pending(step *gherkin.Step, match *StepDef) {
 		step:    step,
 		def:     match,
 		typ:     pending,
+		time:    timeNowFunc(),
 	}
 	f.pending = append(f.pending, s)
+
+	f.features[len(f.features)-1].appendStepResult(s)
 }
 
 func (f *basefmt) Summary() {

fmt_junit.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sort"
+	"sync"
 	"time"
 
 	"github.com/DATA-DOG/godog/gherkin"
@@ -16,149 +18,178 @@ func init() {
 
 func junitFunc(suite string, out io.Writer) Formatter {
 	return &junitFormatter{
-		suite: &junitPackageSuite{
+		basefmt: basefmt{
-			Name:       suite,
+			suiteName: suite,
-			TestSuites: make([]*junitTestSuite, 0),
+			started:   timeNowFunc(),
+			indent:    2,
+			out:       out,
 		},
-		out:     out,
+		lock: new(sync.Mutex),
-		started: timeNowFunc(),
 	}
 }
 
 type junitFormatter struct {
-	suite *junitPackageSuite
+	basefmt
-	out   io.Writer
+	lock *sync.Mutex
-
-	// timing
-	started     time.Time
-	caseStarted time.Time
-	featStarted time.Time
-
-	outline        *gherkin.ScenarioOutline
-	outlineExample int
 }
 
-func (j *junitFormatter) Feature(feature *gherkin.Feature, path string, c []byte) {
+func (f *junitFormatter) Node(n interface{}) {
-	testSuite := &junitTestSuite{
+	f.lock.Lock()
-		TestCases: make([]*junitTestCase, 0),
+	defer f.lock.Unlock()
-		Name:      feature.Name,
+	f.basefmt.Node(n)
-	}
-
-	if len(j.suite.TestSuites) > 0 {
-		j.current().Time = timeNowFunc().Sub(j.featStarted).String()
-	}
-	j.featStarted = timeNowFunc()
-	j.suite.TestSuites = append(j.suite.TestSuites, testSuite)
 }
 
-func (j *junitFormatter) Defined(*gherkin.Step, *StepDef) {
+func (f *junitFormatter) Feature(ft *gherkin.Feature, p string, c []byte) {
-
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.basefmt.Feature(ft, p, c)
 }
 
-func (j *junitFormatter) Node(node interface{}) {
+func (f *junitFormatter) Summary() {
-	suite := j.current()
+	suite := buildJUNITPackageSuite(f.suiteName, f.started, f.features)
-	tcase := &junitTestCase{}
 
-	switch t := node.(type) {
+	_, err := io.WriteString(f.out, xml.Header)
-	case *gherkin.ScenarioOutline:
-		j.outline = t
-		j.outlineExample = 0
-		return
-	case *gherkin.Scenario:
-		tcase.Name = t.Name
-		suite.Tests++
-		j.suite.Tests++
-	case *gherkin.TableRow:
-		j.outlineExample++
-		tcase.Name = fmt.Sprintf("%s #%d", j.outline.Name, j.outlineExample)
-		suite.Tests++
-		j.suite.Tests++
-	default:
-		return
-	}
-	j.caseStarted = timeNowFunc()
-	suite.TestCases = append(suite.TestCases, tcase)
-}
-
-func (j *junitFormatter) Failed(step *gherkin.Step, match *StepDef, err error) {
-	suite := j.current()
-	suite.Failures++
-	j.suite.Failures++
-
-	tcase := suite.current()
-	tcase.Time = timeNowFunc().Sub(j.caseStarted).String()
-	tcase.Status = "failed"
-	tcase.Failure = &junitFailure{
-		Message: fmt.Sprintf("%s %s: %s", step.Type, step.Text, err.Error()),
-	}
-}
-
-func (j *junitFormatter) Passed(step *gherkin.Step, match *StepDef) {
-	suite := j.current()
-
-	tcase := suite.current()
-	tcase.Time = timeNowFunc().Sub(j.caseStarted).String()
-	tcase.Status = "passed"
-}
-
-func (j *junitFormatter) Skipped(step *gherkin.Step, match *StepDef) {
-	suite := j.current()
-
-	tcase := suite.current()
-	tcase.Time = timeNowFunc().Sub(j.caseStarted).String()
-	tcase.Error = append(tcase.Error, &junitError{
-		Type:    "skipped",
-		Message: fmt.Sprintf("%s %s", step.Type, step.Text),
-	})
-}
-
-func (j *junitFormatter) Undefined(step *gherkin.Step, match *StepDef) {
-	suite := j.current()
-	tcase := suite.current()
-	if tcase.Status != "undefined" {
-		// do not count two undefined steps as another error
-		suite.Errors++
-		j.suite.Errors++
-	}
-
-	tcase.Time = timeNowFunc().Sub(j.caseStarted).String()
-	tcase.Status = "undefined"
-	tcase.Error = append(tcase.Error, &junitError{
-		Type:    "undefined",
-		Message: fmt.Sprintf("%s %s", step.Type, step.Text),
-	})
-}
-
-func (j *junitFormatter) Pending(step *gherkin.Step, match *StepDef) {
-	suite := j.current()
-	suite.Errors++
-	j.suite.Errors++
-
-	tcase := suite.current()
-	tcase.Time = timeNowFunc().Sub(j.caseStarted).String()
-	tcase.Status = "pending"
-	tcase.Error = append(tcase.Error, &junitError{
-		Type:    "pending",
-		Message: fmt.Sprintf("%s %s: TODO: write pending definition", step.Type, step.Text),
-	})
-}
-
-func (j *junitFormatter) Summary() {
-	if j.current() != nil {
-		j.current().Time = timeNowFunc().Sub(j.featStarted).String()
-	}
-	j.suite.Time = timeNowFunc().Sub(j.started).String()
-	_, err := io.WriteString(j.out, xml.Header)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "failed to write junit string:", err)
 	}
-	enc := xml.NewEncoder(j.out)
+
+	enc := xml.NewEncoder(f.out)
 	enc.Indent("", s(2))
-	if err = enc.Encode(j.suite); err != nil {
+	if err = enc.Encode(suite); err != nil {
 		fmt.Fprintln(os.Stderr, "failed to write junit xml:", err)
 	}
 }
 
+func (f *junitFormatter) Passed(step *gherkin.Step, match *StepDef) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.basefmt.Passed(step, match)
+}
+
+func (f *junitFormatter) Skipped(step *gherkin.Step, match *StepDef) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.basefmt.Skipped(step, match)
+}
+
+func (f *junitFormatter) Undefined(step *gherkin.Step, match *StepDef) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.basefmt.Undefined(step, match)
+}
+
+func (f *junitFormatter) Failed(step *gherkin.Step, match *StepDef, err error) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.basefmt.Failed(step, match, err)
+}
+
+func (f *junitFormatter) Pending(step *gherkin.Step, match *StepDef) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.basefmt.Pending(step, match)
+}
+
+func (f *junitFormatter) Sync(cf ConcurrentFormatter) {
+	if source, ok := cf.(*junitFormatter); ok {
+		f.lock = source.lock
+	}
+}
+
+func (f *junitFormatter) Copy(cf ConcurrentFormatter) {
+	if source, ok := cf.(*junitFormatter); ok {
+		for _, v := range source.features {
+			f.features = append(f.features, v)
+		}
+		for _, v := range source.failed {
+			f.failed = append(f.failed, v)
+		}
+		for _, v := range source.passed {
+			f.passed = append(f.passed, v)
+		}
+		for _, v := range source.skipped {
+			f.skipped = append(f.skipped, v)
+		}
+		for _, v := range source.undefined {
+			f.undefined = append(f.undefined, v)
+		}
+		for _, v := range source.pending {
+			f.pending = append(f.pending, v)
+		}
+	}
+}
+
+func buildJUNITPackageSuite(suiteName string, startedAt time.Time, features []*feature) junitPackageSuite {
+	suite := junitPackageSuite{
+		Name:       suiteName,
+		TestSuites: make([]*junitTestSuite, len(features)),
+		Time:       timeNowFunc().Sub(startedAt).String(),
+	}
+
+	sort.Sort(sortByName(features))
+
+	for idx, feat := range features {
+		ts := junitTestSuite{
+			Name:      feat.Name,
+			Time:      feat.finishedAt().Sub(feat.startedAt()).String(),
+			TestCases: make([]*junitTestCase, len(feat.Scenarios)),
+		}
+
+		for idx, scenario := range feat.Scenarios {
+			tc := junitTestCase{}
+			tc.Name = scenario.Name
+			tc.Time = scenario.finishedAt().Sub(scenario.startedAt()).String()
+
+			ts.Tests++
+			suite.Tests++
+
+			for _, step := range scenario.Steps {
+				switch step.typ {
+				case passed:
+					tc.Status = passed.String()
+				case failed:
+					tc.Status = failed.String()
+					tc.Failure = &junitFailure{
+						Message: fmt.Sprintf("%s %s: %s", step.step.Type, step.step.Text, step.err),
+					}
+				case skipped:
+					tc.Error = append(tc.Error, &junitError{
+						Type:    "skipped",
+						Message: fmt.Sprintf("%s %s", step.step.Type, step.step.Text),
+					})
+				case undefined:
+					tc.Status = undefined.String()
+					tc.Error = append(tc.Error, &junitError{
+						Type:    "undefined",
+						Message: fmt.Sprintf("%s %s", step.step.Type, step.step.Text),
+					})
+				case pending:
+					tc.Status = pending.String()
+					tc.Error = append(tc.Error, &junitError{
+						Type:    "pending",
+						Message: fmt.Sprintf("%s %s: TODO: write pending definition", step.step.Type, step.step.Text),
+					})
+				}
+			}
+
+			switch tc.Status {
+			case failed.String():
+				ts.Failures++
+				suite.Failures++
+			case undefined.String(), pending.String():
+				ts.Errors++
+				suite.Errors++
+			}
+
+			ts.TestCases[idx] = &tc
+		}
+
+		suite.TestSuites[idx] = &ts
+	}
+
+	return suite
+}
+
 type junitFailure struct {
 	Message string `xml:"message,attr"`
 	Type    string `xml:"type,attr,omitempty"`
@@ -190,10 +221,6 @@ type junitTestSuite struct {
 	TestCases []*junitTestCase
 }
 
-func (ts *junitTestSuite) current() *junitTestCase {
-	return ts.TestCases[len(ts.TestCases)-1]
-}
-
 type junitPackageSuite struct {
 	XMLName    xml.Name `xml:"testsuites"`
 	Name       string   `xml:"name,attr"`
@@ -204,7 +231,3 @@ type junitPackageSuite struct {
 	Time       string   `xml:"time,attr"`
 	TestSuites []*junitTestSuite
 }
-
-func (j *junitFormatter) current() *junitTestSuite {
-	return j.suite.TestSuites[len(j.suite.TestSuites)-1]
-}

fmt_progress.go

@@ -122,3 +122,33 @@ func (f *progress) Pending(step *gherkin.Step, match *StepDef) {
 	f.basefmt.Pending(step, match)
 	f.step(f.pending[len(f.pending)-1])
 }
+
+func (f *progress) Sync(cf ConcurrentFormatter) {
+	if source, ok := cf.(*progress); ok {
+		f.lock = source.lock
+		f.steps = source.steps
+	}
+}
+
+func (f *progress) Copy(cf ConcurrentFormatter) {
+	if source, ok := cf.(*progress); ok {
+		for _, v := range source.features {
+			f.features = append(f.features, v)
+		}
+		for _, v := range source.failed {
+			f.failed = append(f.failed, v)
+		}
+		for _, v := range source.passed {
+			f.passed = append(f.passed, v)
+		}
+		for _, v := range source.skipped {
+			f.skipped = append(f.skipped, v)
+		}
+		for _, v := range source.undefined {
+			f.undefined = append(f.undefined, v)
+		}
+		for _, v := range source.pending {
+			f.pending = append(f.pending, v)
+		}
+	}
+}

go.mod

@@ -1,3 +1,9 @@
 module github.com/DATA-DOG/godog
 
 go 1.12
+
+require (
+	github.com/DATA-DOG/go-txdb v0.1.3
+	github.com/go-sql-driver/mysql v1.5.0
+	github.com/lib/pq v1.3.0 // indirect
+)

run.go

@@ -33,8 +33,8 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool
 	var useFmtCopy bool
 	var copyLock sync.Mutex
 
-	// special mode for progress-formatter
+	// special mode for concurrent-formatter
-	if _, ok := r.fmt.(*progress); ok {
+	if _, ok := r.fmt.(ConcurrentFormatter); ok {
 		useFmtCopy = true
 	}
 
@@ -62,12 +62,11 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool
 				fmtCopy = formatterFn()
 				suite.fmt = fmtCopy
 
-				// sync lock and steps for progress printing
+				concurrentDestFmt, dOk := fmtCopy.(ConcurrentFormatter)
-				if sf, ok := suite.fmt.(*progress); ok {
+				concurrentSourceFmt, sOk := r.fmt.(ConcurrentFormatter)
-					if rf, ok := r.fmt.(*progress); ok {
+
-						sf.lock = rf.lock
+				if dOk && sOk {
-						sf.steps = rf.steps
+					concurrentDestFmt.Sync(concurrentSourceFmt)
-					}
 				}
 			} else {
 				suite.fmt = r.fmt
@@ -80,34 +79,18 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool
 			}
 			if useFmtCopy {
 				copyLock.Lock()
-				dest, dOk := r.fmt.(*progress)
+
-				source, sOk := fmtCopy.(*progress)
+				concurrentDestFmt, dOk := r.fmt.(ConcurrentFormatter)
+				concurrentSourceFmt, sOk := fmtCopy.(ConcurrentFormatter)
 
 				if dOk && sOk {
-					for _, v := range source.features {
+					concurrentDestFmt.Copy(concurrentSourceFmt)
-						dest.features = append(dest.features, v)
-					}
-
-					for _, v := range source.failed {
-						dest.failed = append(dest.failed, v)
-					}
-					for _, v := range source.passed {
-						dest.passed = append(dest.passed, v)
-					}
-					for _, v := range source.skipped {
-						dest.skipped = append(dest.skipped, v)
-					}
-					for _, v := range source.undefined {
-						dest.undefined = append(dest.undefined, v)
-					}
-					for _, v := range source.pending {
-						dest.pending = append(dest.pending, v)
-					}
 				} else if !dOk {
 					panic("cant cast dest formatter to progress-typed")
 				} else if !sOk {
 					panic("cant cast source formatter to progress-typed")
 				}
+
 				copyLock.Unlock()
 			}
 		}(&failed, &ft)
@@ -289,13 +272,11 @@ func Run(suite string, contextInitializer func(suite *Suite)) int {
 
 func supportsConcurrency(format string) bool {
 	switch format {
-	case "events":
+	case "progress", "junit":
-	case "junit":
+		return true
-	case "pretty":
+	case "events", "pretty", "cucumber":
-	case "cucumber":
+		return false
 	default:
-		return true // supports concurrency
+		return false
 	}
-
-	return false // does not support concurrency
 }

run_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/DATA-DOG/godog/colors"
 	"github.com/DATA-DOG/godog/gherkin"
@@ -276,7 +277,7 @@ func TestFeatureFilePathParser(t *testing.T) {
 	}
 }
 
-func TestSucceedWithConcurrencyOption(t *testing.T) {
+func TestSucceedWithProgressAndConcurrencyOption(t *testing.T) {
 	output := new(bytes.Buffer)
 
 	opt := Options{
@@ -312,3 +313,118 @@ func TestSucceedWithConcurrencyOption(t *testing.T) {
 		t.Fatalf("unexpected output: \"%s\"", out)
 	}
 }
+
+func TestSucceedWithJunitAndConcurrencyOption(t *testing.T) {
+	output := new(bytes.Buffer)
+
+	opt := Options{
+		Format:      "junit",
+		NoColors:    true,
+		Paths:       []string{"features"},
+		Concurrency: 2,
+		Output:      output,
+	}
+
+	zeroSecondsString := (0 * time.Second).String()
+
+	expectedOutput := `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="succeed" tests="60" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+  <testsuite name="cucumber json formatter" tests="9" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="Support of Feature Plus Scenario Node" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Feature Plus Scenario Node With Tags" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Feature Plus Scenario Outline" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Feature Plus Scenario Outline With Tags" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Feature Plus Scenario With Steps" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Feature Plus Scenario Outline With Steps" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Comments" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Docstrings" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="Support of Undefined, Pending and Skipped status" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="event stream formatter" tests="3" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="should fire only suite events without any scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should process simple scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should process outline scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="load features" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="load features within path" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="load a specific feature file" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="loaded feature should have a number of scenarios #1" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="loaded feature should have a number of scenarios #2" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="loaded feature should have a number of scenarios #3" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="load a number of feature files" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="run background" tests="3" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="should run background steps" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should skip all consequent steps on failure" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should continue undefined steps" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="run features" tests="11" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="should run a normal feature" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should skip steps after failure" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should skip all scenarios if background fails" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should skip steps after undefined" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should match undefined steps in a row" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should skip steps on pending" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should handle pending step" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should mark undefined steps after pending" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should fail suite if undefined steps follow after the failure" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should fail suite and skip pending step after failed step" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should fail suite and skip next step after failed step" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="run features with nested steps" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="should run passing multistep successfully" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should fail multistep" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should fail nested multistep" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should skip steps after undefined multistep" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should match undefined steps in a row" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should mark undefined steps after pending" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="run outline" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="should run a normal outline" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should continue through examples on failure" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should skip examples on background failure" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should translate step table body" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should translate step doc string argument #1" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should translate step doc string argument #2" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="suite events" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="triggers before scenario event" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="triggers appropriate events for a single scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="triggers appropriate events whole feature" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="triggers appropriate events for two feature files" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should not trigger events on empty feature" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should not trigger events on empty scenarios" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="tag filters" tests="4" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="should filter outline examples by tags" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should filter scenarios by X tag" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should filter scenarios by X tag not having Y" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should filter scenarios having Y and Z tags" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="undefined step snippets" tests="5" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="should generate snippets" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should generate snippets with more arguments" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should handle escaped symbols" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should handle string argument followed by comma" status="passed" time="` + zeroSecondsString + `"></testcase>
+    <testcase name="should handle arguments in the beggining or end of the step" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+  <testsuite name="užkrauti savybes" tests="1" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
+    <testcase name="savybių užkrovimas iš aplanko" status="passed" time="` + zeroSecondsString + `"></testcase>
+  </testsuite>
+</testsuites>`
+
+	status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opt)
+	if status != exitSuccess {
+		t.Fatalf("expected exit status to be 0, but was: %d", status)
+	}
+
+	b, err := ioutil.ReadAll(output)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	out := strings.TrimSpace(string(b))
+	if out != expectedOutput {
+		t.Fatalf("unexpected output: \"%s\"", out)
+	}
+}

suite.go

@@ -12,6 +12,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 	"unicode/utf8"
 
 	"github.com/DATA-DOG/godog/gherkin"
@@ -22,11 +23,58 @@ var typeOfBytes = reflect.TypeOf([]byte(nil))
 
 type feature struct {
 	*gherkin.Feature
+
+	Scenarios []*scenario
+
+	time    time.Time
 	Content []byte `json:"-"`
 	Path    string `json:"path"`
 	order   int
 }
 
+func (f feature) startedAt() time.Time {
+	return f.time
+}
+
+func (f feature) finishedAt() time.Time {
+	if len(f.Scenarios) == 0 {
+		return f.startedAt()
+	}
+
+	return f.Scenarios[len(f.Scenarios)-1].finishedAt()
+}
+
+func (f feature) appendStepResult(s *stepResult) {
+	scenario := f.Scenarios[len(f.Scenarios)-1]
+	scenario.Steps = append(scenario.Steps, s)
+}
+
+type sortByName []*feature
+
+func (s sortByName) Len() int           { return len(s) }
+func (s sortByName) Less(i, j int) bool { return s[i].Name < s[j].Name }
+func (s sortByName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+type scenario struct {
+	Name        string
+	OutlineName string
+	ExampleNo   int
+	time        time.Time
+	Steps       []*stepResult
+}
+
+func (s scenario) startedAt() time.Time {
+	return s.time
+}
+
+func (s scenario) finishedAt() time.Time {
+	if len(s.Steps) == 0 {
+		return s.startedAt()
+	}
+
+	return s.Steps[len(s.Steps)-1].time
+}
+
 // ErrUndefined is returned in case if step definition was not found
 var ErrUndefined = fmt.Errorf("step is undefined")