diff --git a/CHANGELOG.md b/CHANGELOG.md index 4de0b5d..82247c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change LOG +**2017-04-27** +- added an option to randomize scenario execution order, so we could + ensure that scenarios do not depend on global state. + **2016-10-30** - **v0.6.0** - added experimental **events** format, this might be used for unified cucumber formats. But should be not adapted widely, since it is highly diff --git a/README.md b/README.md index abcf71b..cf32125 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ package main import ( "os" "testing" + "time" "github.com/DATA-DOG/godog" ) @@ -217,8 +218,9 @@ func TestMain(m *testing.M) { status := godog.RunWithOptions("godogs", func(s *godog.Suite) { FeatureContext(s) }, godog.Options{ - Format: "progress", - Paths: []string{"features"}, + Format: "progress", + Paths: []string{"features"}, + Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order }) if st := m.Run(); st > status { diff --git a/examples/godogs/godogs_test.go b/examples/godogs/godogs_test.go index 6619e5e..538c43b 100644 --- a/examples/godogs/godogs_test.go +++ b/examples/godogs/godogs_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/DATA-DOG/godog" ) @@ -13,8 +14,9 @@ func TestMain(m *testing.M) { status := godog.RunWithOptions("godogs", func(s *godog.Suite) { FeatureContext(s) }, godog.Options{ - Format: "progress", - Paths: []string{"features"}, + Format: "progress", + Paths: []string{"features"}, + Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order }) if st := m.Run(); st > status { diff --git a/flags.go b/flags.go index ed052c3..3b15e9a 100644 --- a/flags.go +++ b/flags.go @@ -4,7 +4,10 @@ import ( "flag" "fmt" "io" + "math/rand" + "strconv" "strings" + "time" "github.com/DATA-DOG/godog/colors" ) @@ -26,6 +29,10 @@ var descTagsOption = "Filter scenarios by tags. Expression can be:\n" + s(4) + "- " + colors.Yellow(`"@wip && ~@new"`) + ": run wip scenarios, but exclude new\n" + s(4) + "- " + colors.Yellow(`"@wip,@undone"`) + ": run wip or undone scenarios" +var descRandomOption = "Randomly shuffle the scenario execution order.\n" + + "Specify SEED to reproduce the shuffling from a previous run.\n" + + s(4) + `e.g. ` + colors.Yellow(`--random`) + " or " + colors.Yellow(`--random=5738`) + // FlagSet allows to manage flags by external suite runner func FlagSet(opt *Options) *flag.FlagSet { descFormatOption := "How to format tests output. Available formats:\n" @@ -46,6 +53,7 @@ func FlagSet(opt *Options) *flag.FlagSet { set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.") set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.") set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.") + set.Var(&randomSeed{&opt.Randomize}, "random", descRandomOption) set.Usage = usage(set, opt.Output) return set } @@ -64,7 +72,14 @@ func (f *flagged) name() string { case len(f.short) > 0: name = fmt.Sprintf("-%s", f.short) } - if f.dflt != "true" && f.dflt != "false" { + + if f.long == "random" { + // `random` is special in that we will later assign it randomly + // if the user specifies `--random` without specifying one, + // so mask the "default" value here to avoid UI confusion about + // what the value will end up being. + name += "[=SEED]" + } else if f.dflt != "true" && f.dflt != "false" { name += "=" + f.dflt } return name @@ -135,3 +150,42 @@ func usage(set *flag.FlagSet, w io.Writer) func() { fmt.Fprintln(w, "") } } + +// randomSeed implements `flag.Value`, see https://golang.org/pkg/flag/#Value +type randomSeed struct { + ref *int64 +} + +// Choose randomly assigns a convenient pseudo-random seed value. +// The resulting seed will be between `1-99999` for later ease of specification. +func (rs *randomSeed) choose() { + r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) + *rs.ref = r.Int63n(99998) + 1 +} + +func (rs *randomSeed) Set(s string) error { + if s == "true" { + rs.choose() + return nil + } + + if s == "false" { + *rs.ref = 0 + return nil + } + + i, err := strconv.ParseInt(s, 10, 64) + *rs.ref = i + return err +} + +func (rs randomSeed) String() string { + return strconv.FormatInt(*rs.ref, 10) +} + +// If a Value has an IsBoolFlag() bool method returning true, the command-line +// parser makes -name equivalent to -name=true rather than using the next +// command-line argument. +func (rs *randomSeed) IsBoolFlag() bool { + return *rs.ref == 0 +} diff --git a/fmt.go b/fmt.go index aed5a52..dd8ac7c 100644 --- a/fmt.go +++ b/fmt.go @@ -4,8 +4,10 @@ import ( "bytes" "fmt" "io" + "os" "reflect" "regexp" + "strconv" "strings" "text/template" "time" @@ -319,6 +321,13 @@ func (f *basefmt) Summary() { } fmt.Fprintln(f.out, elapsed) + // prints used randomization seed + seed, err := strconv.ParseInt(os.Getenv("GODOG_SEED"), 10, 64) + if err == nil && seed != 0 { + fmt.Fprintln(f.out, "") + fmt.Fprintln(f.out, "Randomized with seed:", colors.Yellow(seed)) + } + if text := f.snippets(); text != "" { fmt.Fprintln(f.out, yellow("\nYou can implement step definitions for undefined steps with these snippets:")) fmt.Fprintln(f.out, yellow(text)) diff --git a/options.go b/options.go index 29a6070..36710c2 100644 --- a/options.go +++ b/options.go @@ -1,6 +1,8 @@ package godog -import "io" +import ( + "io" +) // Options are suite run options // flags are mapped to these options. @@ -13,6 +15,23 @@ type Options struct { // Print step definitions found and exit ShowStepDefinitions bool + // Randomize, if not `0`, will be used to run scenarios in a random order. + // + // Randomizing scenario order is especially helpful for detecting + // situations where you have state leaking between scenarios, which can + // cause flickering or fragile tests. + // + // The default value of `0` means "do not randomize". + // + // The magic value of `-1` means "pick a random seed for me", and godog will + // assign a seed on it's own during the `RunWithOptions` phase, similar to if + // you specified `--random` on the command line. + // + // Any other value will be used as the random seed for shuffling. Re-using the + // same seed will allow you to reproduce the shuffle order of a previous run + // to isolate an error condition. + Randomize int64 + // Stops on the first failure StopOnFailure bool diff --git a/run.go b/run.go index cb73447..9761ecb 100644 --- a/run.go +++ b/run.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "strconv" "github.com/DATA-DOG/godog/colors" ) @@ -11,6 +12,7 @@ import ( type initializer func(*Suite) type runner struct { + randomSeed int64 stopOnFailure bool features []*feature fmt Formatter @@ -30,6 +32,7 @@ func (r *runner) concurrent(rate int) (failed bool) { } suite := &Suite{ fmt: r.fmt, + randomSeed: r.randomSeed, stopOnFailure: r.stopOnFailure, features: []*feature{feat}, } @@ -54,6 +57,7 @@ func (r *runner) concurrent(rate int) (failed bool) { func (r *runner) run() (failed bool) { suite := &Suite{ fmt: r.fmt, + randomSeed: r.randomSeed, stopOnFailure: r.stopOnFailure, features: r.features, } @@ -110,9 +114,13 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt fmt: formatter(suite, output), initializer: contextInitializer, features: features, + randomSeed: opt.Randomize, stopOnFailure: opt.StopOnFailure, } + // store chosen seed in environment, so it could be seen in formatter summary report + os.Setenv("GODOG_SEED", strconv.FormatInt(r.randomSeed, 10)) + var failed bool if opt.Concurrency > 1 { failed = r.concurrent(opt.Concurrency) diff --git a/suite.go b/suite.go index 55fd057..d6c1631 100644 --- a/suite.go +++ b/suite.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "math/rand" "os" "path/filepath" "reflect" @@ -49,6 +50,7 @@ type Suite struct { fmt Formatter failed bool + randomSeed int64 stopOnFailure bool // suite event handlers @@ -330,7 +332,21 @@ func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou func (s *Suite) runFeature(f *feature) { s.fmt.Feature(f.Feature, f.Path, f.Content) - for _, scenario := range f.ScenarioDefinitions { + + // make a local copy of the feature scenario defenitions, + // then shuffle it if we are randomizing scenarios + scenarios := make([]interface{}, len(f.ScenarioDefinitions)) + if s.randomSeed != 0 { + r := rand.New(rand.NewSource(s.randomSeed)) + perm := r.Perm(len(f.ScenarioDefinitions)) + for i, v := range perm { + scenarios[v] = f.ScenarioDefinitions[i] + } + } else { + copy(scenarios, f.ScenarioDefinitions) + } + + for _, scenario := range scenarios { var err error if f.Background != nil { s.fmt.Node(f.Background)