Merge pull request #76 from mroth/randomize-order

allow randomizing scenario order

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change LOG
 
+**2017-04-27**
+- added an option to randomize scenario execution order, so we could
+  ensure that scenarios do not depend on global state.
+
 **2016-10-30** - **v0.6.0**
 - added experimental **events** format, this might be used for unified
   cucumber formats. But should be not adapted widely, since it is highly

README.md

@@ -209,6 +209,7 @@ package main
 import (
 	"os"
 	"testing"
+	"time"
 
 	"github.com/DATA-DOG/godog"
 )
@@ -217,8 +218,9 @@ func TestMain(m *testing.M) {
 	status := godog.RunWithOptions("godogs", func(s *godog.Suite) {
 		FeatureContext(s)
 	}, godog.Options{
-		Format: "progress",
-		Paths:  []string{"features"},
+		Format:    "progress",
+		Paths:     []string{"features"},
+		Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order
 	})
 
 	if st := m.Run(); st > status {

examples/godogs/godogs_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/DATA-DOG/godog"
 )
@@ -13,8 +14,9 @@ func TestMain(m *testing.M) {
 	status := godog.RunWithOptions("godogs", func(s *godog.Suite) {
 		FeatureContext(s)
 	}, godog.Options{
-		Format: "progress",
-		Paths:  []string{"features"},
+		Format:    "progress",
+		Paths:     []string{"features"},
+		Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order
 	})
 
 	if st := m.Run(); st > status {

flags.go

@@ -4,7 +4,10 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"math/rand"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/DATA-DOG/godog/colors"
 )
@@ -26,6 +29,10 @@ var descTagsOption = "Filter scenarios by tags. Expression can be:\n" +
 	s(4) + "- " + colors.Yellow(`"@wip && ~@new"`) + ": run wip scenarios, but exclude new\n" +
 	s(4) + "- " + colors.Yellow(`"@wip,@undone"`) + ": run wip or undone scenarios"
 
+var descRandomOption = "Randomly shuffle the scenario execution order.\n" +
+	"Specify SEED to reproduce the shuffling from a previous run.\n" +
+	s(4) + `e.g. ` + colors.Yellow(`--random`) + " or " + colors.Yellow(`--random=5738`)
+
 // FlagSet allows to manage flags by external suite runner
 func FlagSet(opt *Options) *flag.FlagSet {
 	descFormatOption := "How to format tests output. Available formats:\n"
@@ -46,6 +53,7 @@ func FlagSet(opt *Options) *flag.FlagSet {
 	set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.")
 	set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.")
 	set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.")
+	set.Var(&randomSeed{&opt.Randomize}, "random", descRandomOption)
 	set.Usage = usage(set, opt.Output)
 	return set
 }
@@ -64,7 +72,14 @@ func (f *flagged) name() string {
 	case len(f.short) > 0:
 		name = fmt.Sprintf("-%s", f.short)
 	}
-	if f.dflt != "true" && f.dflt != "false" {
+
+	if f.long == "random" {
+		// `random` is special in that we will later assign it randomly
+		// if the user specifies `--random` without specifying one,
+		// so mask the "default" value here to avoid UI confusion about
+		// what the value will end up being.
+		name += "[=SEED]"
+	} else if f.dflt != "true" && f.dflt != "false" {
 		name += "=" + f.dflt
 	}
 	return name
@@ -135,3 +150,42 @@ func usage(set *flag.FlagSet, w io.Writer) func() {
 		fmt.Fprintln(w, "")
 	}
 }
+
+// randomSeed implements `flag.Value`, see https://golang.org/pkg/flag/#Value
+type randomSeed struct {
+	ref *int64
+}
+
+// Choose randomly assigns a convenient pseudo-random seed value.
+// The resulting seed will be between `1-99999` for later ease of specification.
+func (rs *randomSeed) choose() {
+	r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
+	*rs.ref = r.Int63n(99998) + 1
+}
+
+func (rs *randomSeed) Set(s string) error {
+	if s == "true" {
+		rs.choose()
+		return nil
+	}
+
+	if s == "false" {
+		*rs.ref = 0
+		return nil
+	}
+
+	i, err := strconv.ParseInt(s, 10, 64)
+	*rs.ref = i
+	return err
+}
+
+func (rs randomSeed) String() string {
+	return strconv.FormatInt(*rs.ref, 10)
+}
+
+// If a Value has an IsBoolFlag() bool method returning true, the command-line
+// parser makes -name equivalent to -name=true rather than using the next
+// command-line argument.
+func (rs *randomSeed) IsBoolFlag() bool {
+	return *rs.ref == 0
+}

fmt.go

@@ -4,8 +4,10 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"os"
 	"reflect"
 	"regexp"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -319,6 +321,13 @@ func (f *basefmt) Summary() {
 	}
 	fmt.Fprintln(f.out, elapsed)
 
+	// prints used randomization seed
+	seed, err := strconv.ParseInt(os.Getenv("GODOG_SEED"), 10, 64)
+	if err == nil && seed != 0 {
+		fmt.Fprintln(f.out, "")
+		fmt.Fprintln(f.out, "Randomized with seed:", colors.Yellow(seed))
+	}
+
 	if text := f.snippets(); text != "" {
 		fmt.Fprintln(f.out, yellow("\nYou can implement step definitions for undefined steps with these snippets:"))
 		fmt.Fprintln(f.out, yellow(text))

options.go

@@ -1,6 +1,8 @@
 package godog
 
-import "io"
+import (
+	"io"
+)
 
 // Options are suite run options
 // flags are mapped to these options.
@@ -13,6 +15,23 @@ type Options struct {
 	// Print step definitions found and exit
 	ShowStepDefinitions bool
 
+	// Randomize, if not `0`, will be used to run scenarios in a random order.
+	//
+	// Randomizing scenario order is especially helpful for detecting
+	// situations where you have state leaking between scenarios, which can
+	// cause flickering or fragile tests.
+	//
+	// The default value of `0` means "do not randomize".
+	//
+	// The magic value of `-1` means "pick a random seed for me", and godog will
+	// assign a seed on it's own during the `RunWithOptions` phase, similar to if
+	// you specified `--random` on the command line.
+	//
+	// Any other value will be used as the random seed for shuffling. Re-using the
+	// same seed will allow you to reproduce the shuffle order of a previous run
+	// to isolate an error condition.
+	Randomize int64
+
 	// Stops on the first failure
 	StopOnFailure bool
 

run.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 
 	"github.com/DATA-DOG/godog/colors"
 )
@@ -11,6 +12,7 @@ import (
 type initializer func(*Suite)
 
 type runner struct {
+	randomSeed    int64
 	stopOnFailure bool
 	features      []*feature
 	fmt           Formatter
@@ -30,6 +32,7 @@ func (r *runner) concurrent(rate int) (failed bool) {
 			}
 			suite := &Suite{
 				fmt:           r.fmt,
+				randomSeed:    r.randomSeed,
 				stopOnFailure: r.stopOnFailure,
 				features:      []*feature{feat},
 			}
@@ -54,6 +57,7 @@ func (r *runner) concurrent(rate int) (failed bool) {
 func (r *runner) run() (failed bool) {
 	suite := &Suite{
 		fmt:           r.fmt,
+		randomSeed:    r.randomSeed,
 		stopOnFailure: r.stopOnFailure,
 		features:      r.features,
 	}
@@ -110,9 +114,13 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt
 		fmt:           formatter(suite, output),
 		initializer:   contextInitializer,
 		features:      features,
+		randomSeed:    opt.Randomize,
 		stopOnFailure: opt.StopOnFailure,
 	}
 
+	// store chosen seed in environment, so it could be seen in formatter summary report
+	os.Setenv("GODOG_SEED", strconv.FormatInt(r.randomSeed, 10))
+
 	var failed bool
 	if opt.Concurrency > 1 {
 		failed = r.concurrent(opt.Concurrency)

suite.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -49,6 +50,7 @@ type Suite struct {
 	fmt      Formatter
 
 	failed        bool
+	randomSeed    int64
 	stopOnFailure bool
 
 	// suite event handlers
@@ -330,7 +332,21 @@ func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou
 
 func (s *Suite) runFeature(f *feature) {
 	s.fmt.Feature(f.Feature, f.Path, f.Content)
-	for _, scenario := range f.ScenarioDefinitions {
+
+	// make a local copy of the feature scenario defenitions,
+	// then shuffle it if we are randomizing scenarios
+	scenarios := make([]interface{}, len(f.ScenarioDefinitions))
+	if s.randomSeed != 0 {
+		r := rand.New(rand.NewSource(s.randomSeed))
+		perm := r.Perm(len(f.ScenarioDefinitions))
+		for i, v := range perm {
+			scenarios[v] = f.ScenarioDefinitions[i]
+		}
+	} else {
+		copy(scenarios, f.ScenarioDefinitions)
+	}
+
+	for _, scenario := range scenarios {
 		var err error
 		if f.Background != nil {
 			s.fmt.Node(f.Background)