diff --git a/CHANGELOG.md b/CHANGELOG.md index 4de0b5d..82247c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change LOG +**2017-04-27** +- added an option to randomize scenario execution order, so we could + ensure that scenarios do not depend on global state. + **2016-10-30** - **v0.6.0** - added experimental **events** format, this might be used for unified cucumber formats. But should be not adapted widely, since it is highly diff --git a/README.md b/README.md index abcf71b..cf32125 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ package main import ( "os" "testing" + "time" "github.com/DATA-DOG/godog" ) @@ -217,8 +218,9 @@ func TestMain(m *testing.M) { status := godog.RunWithOptions("godogs", func(s *godog.Suite) { FeatureContext(s) }, godog.Options{ - Format: "progress", - Paths: []string{"features"}, + Format: "progress", + Paths: []string{"features"}, + Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order }) if st := m.Run(); st > status { diff --git a/examples/godogs/godogs_test.go b/examples/godogs/godogs_test.go index 6619e5e..538c43b 100644 --- a/examples/godogs/godogs_test.go +++ b/examples/godogs/godogs_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/DATA-DOG/godog" ) @@ -13,8 +14,9 @@ func TestMain(m *testing.M) { status := godog.RunWithOptions("godogs", func(s *godog.Suite) { FeatureContext(s) }, godog.Options{ - Format: "progress", - Paths: []string{"features"}, + Format: "progress", + Paths: []string{"features"}, + Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order }) if st := m.Run(); st > status { diff --git a/flags.go b/flags.go index b56b3a9..3b15e9a 100644 --- a/flags.go +++ b/flags.go @@ -4,7 +4,10 @@ import ( "flag" "fmt" "io" + "math/rand" + "strconv" "strings" + "time" "github.com/DATA-DOG/godog/colors" ) @@ -50,7 +53,7 @@ func FlagSet(opt *Options) *flag.FlagSet { set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.") set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.") set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.") - set.Var(&opt.Randomize, "random", descRandomOption) + set.Var(&randomSeed{&opt.Randomize}, "random", descRandomOption) set.Usage = usage(set, opt.Output) return set } @@ -147,3 +150,42 @@ func usage(set *flag.FlagSet, w io.Writer) func() { fmt.Fprintln(w, "") } } + +// randomSeed implements `flag.Value`, see https://golang.org/pkg/flag/#Value +type randomSeed struct { + ref *int64 +} + +// Choose randomly assigns a convenient pseudo-random seed value. +// The resulting seed will be between `1-99999` for later ease of specification. +func (rs *randomSeed) choose() { + r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) + *rs.ref = r.Int63n(99998) + 1 +} + +func (rs *randomSeed) Set(s string) error { + if s == "true" { + rs.choose() + return nil + } + + if s == "false" { + *rs.ref = 0 + return nil + } + + i, err := strconv.ParseInt(s, 10, 64) + *rs.ref = i + return err +} + +func (rs randomSeed) String() string { + return strconv.FormatInt(*rs.ref, 10) +} + +// If a Value has an IsBoolFlag() bool method returning true, the command-line +// parser makes -name equivalent to -name=true rather than using the next +// command-line argument. +func (rs *randomSeed) IsBoolFlag() bool { + return *rs.ref == 0 +} diff --git a/fmt.go b/fmt.go index aed5a52..dd8ac7c 100644 --- a/fmt.go +++ b/fmt.go @@ -4,8 +4,10 @@ import ( "bytes" "fmt" "io" + "os" "reflect" "regexp" + "strconv" "strings" "text/template" "time" @@ -319,6 +321,13 @@ func (f *basefmt) Summary() { } fmt.Fprintln(f.out, elapsed) + // prints used randomization seed + seed, err := strconv.ParseInt(os.Getenv("GODOG_SEED"), 10, 64) + if err == nil && seed != 0 { + fmt.Fprintln(f.out, "") + fmt.Fprintln(f.out, "Randomized with seed:", colors.Yellow(seed)) + } + if text := f.snippets(); text != "" { fmt.Fprintln(f.out, yellow("\nYou can implement step definitions for undefined steps with these snippets:")) fmt.Fprintln(f.out, yellow(text)) diff --git a/options.go b/options.go index b5899d9..36710c2 100644 --- a/options.go +++ b/options.go @@ -2,9 +2,6 @@ package godog import ( "io" - "math/rand" - "strconv" - "time" ) // Options are suite run options @@ -33,7 +30,7 @@ type Options struct { // Any other value will be used as the random seed for shuffling. Re-using the // same seed will allow you to reproduce the shuffle order of a previous run // to isolate an error condition. - Randomize randomSeed + Randomize int64 // Stops on the first failure StopOnFailure bool @@ -57,40 +54,3 @@ type Options struct { // Where it should print formatter output Output io.Writer } - -// randomSeed implements `flag.Value`, see https://golang.org/pkg/flag/#Value -type randomSeed int64 - -// Choose randomly assigns a convenient pseudo-random seed value. -// The resulting seed will be between `1-99999` for later ease of specification. -func (rs *randomSeed) Choose() { - r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) - *rs = randomSeed(r.Int63n(99998) + 1) -} - -func (rs *randomSeed) Set(s string) error { - if s == "true" { - rs.Choose() - return nil - } - - if s == "false" { - *rs = 0 - return nil - } - - i, err := strconv.ParseInt(s, 10, 64) - *rs = randomSeed(i) - return err -} - -func (rs randomSeed) String() string { - return strconv.FormatInt(int64(rs), 10) -} - -// If a Value has an IsBoolFlag() bool method returning true, the command-line -// parser makes -name equivalent to -name=true rather than using the next -// command-line argument. -func (rs *randomSeed) IsBoolFlag() bool { - return *rs == 0 -} diff --git a/run.go b/run.go index 0aca7f8..9761ecb 100644 --- a/run.go +++ b/run.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "strconv" "github.com/DATA-DOG/godog/colors" ) @@ -109,20 +110,17 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt features, err := parseFeatures(opt.Tags, opt.Paths) fatal(err) - // handle convenience condition where user specified `-1` in Options struct, - // asking us to choose the random value seed for them. - if opt.Randomize == -1 { - opt.Randomize.Choose() - } - r := runner{ fmt: formatter(suite, output), initializer: contextInitializer, features: features, - randomSeed: int64(opt.Randomize), + randomSeed: opt.Randomize, stopOnFailure: opt.StopOnFailure, } + // store chosen seed in environment, so it could be seen in formatter summary report + os.Setenv("GODOG_SEED", strconv.FormatInt(r.randomSeed, 10)) + var failed bool if opt.Concurrency > 1 { failed = r.concurrent(opt.Concurrency)