From 4a754b26a5c1fd9c15543f7273810c38ab2f1f90 Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Mon, 24 Apr 2017 21:10:55 -0400 Subject: [PATCH 1/6] allow randomizing scenario order --- options.go | 7 +++++++ run.go | 12 ++++++++++++ suite.go | 17 ++++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/options.go b/options.go index 29a6070..5858428 100644 --- a/options.go +++ b/options.go @@ -13,6 +13,13 @@ type Options struct { // Print step definitions found and exit ShowStepDefinitions bool + // Run scenarios in random order. + // + // This is especially helpful for detecting situations + // where you have state leaking between scenarios, which + // can cause flickering or fragile tests. + RandomOrder bool + // Stops on the first failure StopOnFailure bool diff --git a/run.go b/run.go index cb73447..50f93b2 100644 --- a/run.go +++ b/run.go @@ -3,7 +3,9 @@ package godog import ( "fmt" "io" + "math/rand" "os" + "time" "github.com/DATA-DOG/godog/colors" ) @@ -11,6 +13,7 @@ import ( type initializer func(*Suite) type runner struct { + randomOrder bool stopOnFailure bool features []*feature fmt Formatter @@ -30,6 +33,7 @@ func (r *runner) concurrent(rate int) (failed bool) { } suite := &Suite{ fmt: r.fmt, + randomOrder: r.randomOrder, stopOnFailure: r.stopOnFailure, features: []*feature{feat}, } @@ -54,6 +58,7 @@ func (r *runner) concurrent(rate int) (failed bool) { func (r *runner) run() (failed bool) { suite := &Suite{ fmt: r.fmt, + randomOrder: r.randomOrder, stopOnFailure: r.stopOnFailure, features: r.features, } @@ -110,9 +115,16 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt fmt: formatter(suite, output), initializer: contextInitializer, features: features, + randomOrder: opt.RandomOrder, stopOnFailure: opt.StopOnFailure, } + if opt.RandomOrder { + // TODO(mroth): allow for seed to be specified in options, + // and print it at the end of a run for replication purposes + rand.Seed(time.Now().UTC().UnixNano()) + } + var failed bool if opt.Concurrency > 1 { failed = r.concurrent(opt.Concurrency) diff --git a/suite.go b/suite.go index 55fd057..18cd7f8 100644 --- a/suite.go +++ b/suite.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "math/rand" "os" "path/filepath" "reflect" @@ -49,6 +50,7 @@ type Suite struct { fmt Formatter failed bool + randomOrder bool stopOnFailure bool // suite event handlers @@ -330,7 +332,20 @@ func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Backgrou func (s *Suite) runFeature(f *feature) { s.fmt.Feature(f.Feature, f.Path, f.Content) - for _, scenario := range f.ScenarioDefinitions { + + // make a local copy of the feature scenario defenitions, + // then shuffle it if we are randomizing scenarios + scenarios := make([]interface{}, len(f.ScenarioDefinitions)) + if s.randomOrder { + perm := rand.Perm(len(f.ScenarioDefinitions)) + for i, v := range perm { + scenarios[v] = f.ScenarioDefinitions[i] + } + } else { + copy(scenarios, f.ScenarioDefinitions) + } + + for _, scenario := range scenarios { var err error if f.Background != nil { s.fmt.Node(f.Background) From 0eed963c6323410f50d753551b0f17e3a57213f4 Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Tue, 25 Apr 2017 11:42:40 -0400 Subject: [PATCH 2/6] switch Random opt from bool to int64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows for us to propagate the seed value around, and not have to have two different values in Options when we want to allow specification of seed. It does introduce some “magic values”, but the document them in the Options struct docstrings. --- options.go | 15 ++++++++++----- run.go | 26 ++++++++++++++++---------- suite.go | 4 ++-- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/options.go b/options.go index 5858428..a1235ee 100644 --- a/options.go +++ b/options.go @@ -13,12 +13,17 @@ type Options struct { // Print step definitions found and exit ShowStepDefinitions bool - // Run scenarios in random order. + // RandomSeed, if not `0`, will be used to run scenarios in a random order. // - // This is especially helpful for detecting situations - // where you have state leaking between scenarios, which - // can cause flickering or fragile tests. - RandomOrder bool + // Randomizing scenario order is especially helpful for detecting + // situations where you have state leaking between scenarios, which can + // cause flickering or fragile tests. + // + // The default value of `0` means "do not randomize". + // + // The magic value of `-1` means "pick a random seed for me", the resulting + // seed will only be between `1-99999` for ease of specification. + RandomSeed int64 // Stops on the first failure StopOnFailure bool diff --git a/run.go b/run.go index 50f93b2..bae334a 100644 --- a/run.go +++ b/run.go @@ -13,7 +13,7 @@ import ( type initializer func(*Suite) type runner struct { - randomOrder bool + randomSeed int64 stopOnFailure bool features []*feature fmt Formatter @@ -33,7 +33,7 @@ func (r *runner) concurrent(rate int) (failed bool) { } suite := &Suite{ fmt: r.fmt, - randomOrder: r.randomOrder, + randomSeed: r.randomSeed, stopOnFailure: r.stopOnFailure, features: []*feature{feat}, } @@ -58,7 +58,7 @@ func (r *runner) concurrent(rate int) (failed bool) { func (r *runner) run() (failed bool) { suite := &Suite{ fmt: r.fmt, - randomOrder: r.randomOrder, + randomSeed: r.randomSeed, stopOnFailure: r.stopOnFailure, features: r.features, } @@ -111,20 +111,26 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt features, err := parseFeatures(opt.Tags, opt.Paths) fatal(err) + seed := opt.RandomSeed + if seed == -1 { + // if RandomSeed opt is -1, means pick a memorable rand seed as default + // the docStrings for Option specify this should be 1-99999 (same as ruby Cucumber) + r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) + seed = r.Int63n(99998) + 1 + } + if seed != 0 { + // init global rand module (concurrent safe) with our seed + rand.Seed(seed) + } + r := runner{ fmt: formatter(suite, output), initializer: contextInitializer, features: features, - randomOrder: opt.RandomOrder, + randomSeed: seed, stopOnFailure: opt.StopOnFailure, } - if opt.RandomOrder { - // TODO(mroth): allow for seed to be specified in options, - // and print it at the end of a run for replication purposes - rand.Seed(time.Now().UTC().UnixNano()) - } - var failed bool if opt.Concurrency > 1 { failed = r.concurrent(opt.Concurrency) diff --git a/suite.go b/suite.go index 18cd7f8..e386f75 100644 --- a/suite.go +++ b/suite.go @@ -50,7 +50,7 @@ type Suite struct { fmt Formatter failed bool - randomOrder bool + randomSeed int64 stopOnFailure bool // suite event handlers @@ -336,7 +336,7 @@ func (s *Suite) runFeature(f *feature) { // make a local copy of the feature scenario defenitions, // then shuffle it if we are randomizing scenarios scenarios := make([]interface{}, len(f.ScenarioDefinitions)) - if s.randomOrder { + if s.randomSeed != 0 { perm := rand.Perm(len(f.ScenarioDefinitions)) for i, v := range perm { scenarios[v] = f.ScenarioDefinitions[i] From cf6160b3f2f4b0b3435a3d5609bcd93ddabe2e15 Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Tue, 25 Apr 2017 12:33:20 -0400 Subject: [PATCH 3/6] fix deterministic scenario ordering with feature concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit while global rand is concurrent safe, since each suite is handling picking scenario order on it’s own, if running features under concurrency we cannot guarantee the order Features will execute under, therefore we have each suite initialize its own randomness Source with the initial seed, such that their random order selection within a Feature should always be deterministic when given a specific seed. --- run.go | 4 ---- suite.go | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/run.go b/run.go index bae334a..7d2351c 100644 --- a/run.go +++ b/run.go @@ -118,10 +118,6 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) seed = r.Int63n(99998) + 1 } - if seed != 0 { - // init global rand module (concurrent safe) with our seed - rand.Seed(seed) - } r := runner{ fmt: formatter(suite, output), diff --git a/suite.go b/suite.go index e386f75..d6c1631 100644 --- a/suite.go +++ b/suite.go @@ -337,7 +337,8 @@ func (s *Suite) runFeature(f *feature) { // then shuffle it if we are randomizing scenarios scenarios := make([]interface{}, len(f.ScenarioDefinitions)) if s.randomSeed != 0 { - perm := rand.Perm(len(f.ScenarioDefinitions)) + r := rand.New(rand.NewSource(s.randomSeed)) + perm := r.Perm(len(f.ScenarioDefinitions)) for i, v := range perm { scenarios[v] = f.ScenarioDefinitions[i] } From f50c3b5e1434ef0ef1414787d153b69b3becb1e9 Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Tue, 25 Apr 2017 14:50:58 -0400 Subject: [PATCH 4/6] allow setting randomization via flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allows for setting randomization with flags, but also modifies the Options struct to now have two separate values Overall this relates in a much more understandable UI from a user perspective, since the normal case for randomization will be a user just saying “run these out of order please” — and in a rare case they may wish to additionally supply a seed to try to reproduce an older run to trigger the same error condition. --- flags.go | 11 ++++++++++- options.go | 14 ++++++++++---- run.go | 12 ++++++++---- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/flags.go b/flags.go index ed052c3..a839a5f 100644 --- a/flags.go +++ b/flags.go @@ -46,6 +46,8 @@ func FlagSet(opt *Options) *flag.FlagSet { set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.") set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.") set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.") + set.BoolVar(&opt.Randomize, "random", false, "Randomize scenario execution order.") + set.Int64Var(&opt.RandomSeed, "seed", 0, "Specify random seed to reproduce shuffling from a previous run.\n(implies --random)") set.Usage = usage(set, opt.Output) return set } @@ -64,7 +66,14 @@ func (f *flagged) name() string { case len(f.short) > 0: name = fmt.Sprintf("-%s", f.short) } - if f.dflt != "true" && f.dflt != "false" { + + if f.long == "seed" { + // `seed`` is special in that we will later assign it randomly + // if the user specifies `--random` without specifying one, + // so mask the "default" value here to avoid UI confusion about + // what the value will end up being. + name += "=SEED" + } else if f.dflt != "true" && f.dflt != "false" { name += "=" + f.dflt } return name diff --git a/options.go b/options.go index a1235ee..b1d9625 100644 --- a/options.go +++ b/options.go @@ -13,16 +13,22 @@ type Options struct { // Print step definitions found and exit ShowStepDefinitions bool - // RandomSeed, if not `0`, will be used to run scenarios in a random order. + // Randomize causes scenarios to be run in random order. // // Randomizing scenario order is especially helpful for detecting // situations where you have state leaking between scenarios, which can // cause flickering or fragile tests. + Randomize bool + + // RandomSeed allows specifying the seed to reproduce the random scenario + // shuffling from a previous run. // - // The default value of `0` means "do not randomize". + // When `RandomSeed` is left at the nil value (`0`), but `Randomize` + // has been set to `true`, then godog will automatically pick a random + // seed between `1-99999` for ease of specification. // - // The magic value of `-1` means "pick a random seed for me", the resulting - // seed will only be between `1-99999` for ease of specification. + // If RandomSeed is set to anything other than the default nil value (`0`), + // then `Randomize = true` will be implied. RandomSeed int64 // Stops on the first failure diff --git a/run.go b/run.go index 7d2351c..a20fd34 100644 --- a/run.go +++ b/run.go @@ -111,10 +111,14 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt features, err := parseFeatures(opt.Tags, opt.Paths) fatal(err) - seed := opt.RandomSeed - if seed == -1 { - // if RandomSeed opt is -1, means pick a memorable rand seed as default - // the docStrings for Option specify this should be 1-99999 (same as ruby Cucumber) + // the actual seed value which will be propogated + // if left as nil value, then scenarios run sequentially + var seed int64 + // use specified seed if exists, or assign one ourselves if + // none specified but user wants randomization + if opt.RandomSeed != 0 { + seed = opt.RandomSeed + } else if opt.Randomize && opt.RandomSeed == 0 { r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) seed = r.Int63n(99998) + 1 } From d2eb563953e019cc6939fd27b3c7649f467d59b1 Mon Sep 17 00:00:00 2001 From: Matthew Rothenberg Date: Wed, 26 Apr 2017 12:21:12 -0400 Subject: [PATCH 5/6] single randomSeed Option value, handled as flag.Value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Best of both worlds, this allows for a single int64 value in the Options struct, but at the same time we can do `—random` from command line with no args and have a default value Chosen. --- flags.go | 13 ++++++----- options.go | 65 ++++++++++++++++++++++++++++++++++++++++++++---------- run.go | 18 +++++---------- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/flags.go b/flags.go index a839a5f..b56b3a9 100644 --- a/flags.go +++ b/flags.go @@ -26,6 +26,10 @@ var descTagsOption = "Filter scenarios by tags. Expression can be:\n" + s(4) + "- " + colors.Yellow(`"@wip && ~@new"`) + ": run wip scenarios, but exclude new\n" + s(4) + "- " + colors.Yellow(`"@wip,@undone"`) + ": run wip or undone scenarios" +var descRandomOption = "Randomly shuffle the scenario execution order.\n" + + "Specify SEED to reproduce the shuffling from a previous run.\n" + + s(4) + `e.g. ` + colors.Yellow(`--random`) + " or " + colors.Yellow(`--random=5738`) + // FlagSet allows to manage flags by external suite runner func FlagSet(opt *Options) *flag.FlagSet { descFormatOption := "How to format tests output. Available formats:\n" @@ -46,8 +50,7 @@ func FlagSet(opt *Options) *flag.FlagSet { set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.") set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.") set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.") - set.BoolVar(&opt.Randomize, "random", false, "Randomize scenario execution order.") - set.Int64Var(&opt.RandomSeed, "seed", 0, "Specify random seed to reproduce shuffling from a previous run.\n(implies --random)") + set.Var(&opt.Randomize, "random", descRandomOption) set.Usage = usage(set, opt.Output) return set } @@ -67,12 +70,12 @@ func (f *flagged) name() string { name = fmt.Sprintf("-%s", f.short) } - if f.long == "seed" { - // `seed`` is special in that we will later assign it randomly + if f.long == "random" { + // `random` is special in that we will later assign it randomly // if the user specifies `--random` without specifying one, // so mask the "default" value here to avoid UI confusion about // what the value will end up being. - name += "=SEED" + name += "[=SEED]" } else if f.dflt != "true" && f.dflt != "false" { name += "=" + f.dflt } diff --git a/options.go b/options.go index b1d9625..b5899d9 100644 --- a/options.go +++ b/options.go @@ -1,6 +1,11 @@ package godog -import "io" +import ( + "io" + "math/rand" + "strconv" + "time" +) // Options are suite run options // flags are mapped to these options. @@ -13,23 +18,22 @@ type Options struct { // Print step definitions found and exit ShowStepDefinitions bool - // Randomize causes scenarios to be run in random order. + // Randomize, if not `0`, will be used to run scenarios in a random order. // // Randomizing scenario order is especially helpful for detecting // situations where you have state leaking between scenarios, which can // cause flickering or fragile tests. - Randomize bool - - // RandomSeed allows specifying the seed to reproduce the random scenario - // shuffling from a previous run. // - // When `RandomSeed` is left at the nil value (`0`), but `Randomize` - // has been set to `true`, then godog will automatically pick a random - // seed between `1-99999` for ease of specification. + // The default value of `0` means "do not randomize". // - // If RandomSeed is set to anything other than the default nil value (`0`), - // then `Randomize = true` will be implied. - RandomSeed int64 + // The magic value of `-1` means "pick a random seed for me", and godog will + // assign a seed on it's own during the `RunWithOptions` phase, similar to if + // you specified `--random` on the command line. + // + // Any other value will be used as the random seed for shuffling. Re-using the + // same seed will allow you to reproduce the shuffle order of a previous run + // to isolate an error condition. + Randomize randomSeed // Stops on the first failure StopOnFailure bool @@ -53,3 +57,40 @@ type Options struct { // Where it should print formatter output Output io.Writer } + +// randomSeed implements `flag.Value`, see https://golang.org/pkg/flag/#Value +type randomSeed int64 + +// Choose randomly assigns a convenient pseudo-random seed value. +// The resulting seed will be between `1-99999` for later ease of specification. +func (rs *randomSeed) Choose() { + r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) + *rs = randomSeed(r.Int63n(99998) + 1) +} + +func (rs *randomSeed) Set(s string) error { + if s == "true" { + rs.Choose() + return nil + } + + if s == "false" { + *rs = 0 + return nil + } + + i, err := strconv.ParseInt(s, 10, 64) + *rs = randomSeed(i) + return err +} + +func (rs randomSeed) String() string { + return strconv.FormatInt(int64(rs), 10) +} + +// If a Value has an IsBoolFlag() bool method returning true, the command-line +// parser makes -name equivalent to -name=true rather than using the next +// command-line argument. +func (rs *randomSeed) IsBoolFlag() bool { + return *rs == 0 +} diff --git a/run.go b/run.go index a20fd34..0aca7f8 100644 --- a/run.go +++ b/run.go @@ -3,9 +3,7 @@ package godog import ( "fmt" "io" - "math/rand" "os" - "time" "github.com/DATA-DOG/godog/colors" ) @@ -111,23 +109,17 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt features, err := parseFeatures(opt.Tags, opt.Paths) fatal(err) - // the actual seed value which will be propogated - // if left as nil value, then scenarios run sequentially - var seed int64 - // use specified seed if exists, or assign one ourselves if - // none specified but user wants randomization - if opt.RandomSeed != 0 { - seed = opt.RandomSeed - } else if opt.Randomize && opt.RandomSeed == 0 { - r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) - seed = r.Int63n(99998) + 1 + // handle convenience condition where user specified `-1` in Options struct, + // asking us to choose the random value seed for them. + if opt.Randomize == -1 { + opt.Randomize.Choose() } r := runner{ fmt: formatter(suite, output), initializer: contextInitializer, features: features, - randomSeed: seed, + randomSeed: int64(opt.Randomize), stopOnFailure: opt.StopOnFailure, } From 201677e1529e1320542ef3d943db0c71688f898f Mon Sep 17 00:00:00 2001 From: gedi Date: Thu, 27 Apr 2017 11:06:44 +0300 Subject: [PATCH 6/6] prints random seed in formatter summary --- CHANGELOG.md | 4 ++++ README.md | 6 +++-- examples/godogs/godogs_test.go | 6 +++-- flags.go | 44 +++++++++++++++++++++++++++++++++- fmt.go | 9 +++++++ options.go | 42 +------------------------------- run.go | 12 ++++------ 7 files changed, 70 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4de0b5d..82247c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change LOG +**2017-04-27** +- added an option to randomize scenario execution order, so we could + ensure that scenarios do not depend on global state. + **2016-10-30** - **v0.6.0** - added experimental **events** format, this might be used for unified cucumber formats. But should be not adapted widely, since it is highly diff --git a/README.md b/README.md index abcf71b..cf32125 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ package main import ( "os" "testing" + "time" "github.com/DATA-DOG/godog" ) @@ -217,8 +218,9 @@ func TestMain(m *testing.M) { status := godog.RunWithOptions("godogs", func(s *godog.Suite) { FeatureContext(s) }, godog.Options{ - Format: "progress", - Paths: []string{"features"}, + Format: "progress", + Paths: []string{"features"}, + Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order }) if st := m.Run(); st > status { diff --git a/examples/godogs/godogs_test.go b/examples/godogs/godogs_test.go index 6619e5e..538c43b 100644 --- a/examples/godogs/godogs_test.go +++ b/examples/godogs/godogs_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/DATA-DOG/godog" ) @@ -13,8 +14,9 @@ func TestMain(m *testing.M) { status := godog.RunWithOptions("godogs", func(s *godog.Suite) { FeatureContext(s) }, godog.Options{ - Format: "progress", - Paths: []string{"features"}, + Format: "progress", + Paths: []string{"features"}, + Randomize: time.Now().UTC().UnixNano(), // randomize scenario execution order }) if st := m.Run(); st > status { diff --git a/flags.go b/flags.go index b56b3a9..3b15e9a 100644 --- a/flags.go +++ b/flags.go @@ -4,7 +4,10 @@ import ( "flag" "fmt" "io" + "math/rand" + "strconv" "strings" + "time" "github.com/DATA-DOG/godog/colors" ) @@ -50,7 +53,7 @@ func FlagSet(opt *Options) *flag.FlagSet { set.BoolVar(&opt.ShowStepDefinitions, "d", false, "Print all available step definitions.") set.BoolVar(&opt.StopOnFailure, "stop-on-failure", false, "Stop processing on first failed scenario.") set.BoolVar(&opt.NoColors, "no-colors", false, "Disable ansi colors.") - set.Var(&opt.Randomize, "random", descRandomOption) + set.Var(&randomSeed{&opt.Randomize}, "random", descRandomOption) set.Usage = usage(set, opt.Output) return set } @@ -147,3 +150,42 @@ func usage(set *flag.FlagSet, w io.Writer) func() { fmt.Fprintln(w, "") } } + +// randomSeed implements `flag.Value`, see https://golang.org/pkg/flag/#Value +type randomSeed struct { + ref *int64 +} + +// Choose randomly assigns a convenient pseudo-random seed value. +// The resulting seed will be between `1-99999` for later ease of specification. +func (rs *randomSeed) choose() { + r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) + *rs.ref = r.Int63n(99998) + 1 +} + +func (rs *randomSeed) Set(s string) error { + if s == "true" { + rs.choose() + return nil + } + + if s == "false" { + *rs.ref = 0 + return nil + } + + i, err := strconv.ParseInt(s, 10, 64) + *rs.ref = i + return err +} + +func (rs randomSeed) String() string { + return strconv.FormatInt(*rs.ref, 10) +} + +// If a Value has an IsBoolFlag() bool method returning true, the command-line +// parser makes -name equivalent to -name=true rather than using the next +// command-line argument. +func (rs *randomSeed) IsBoolFlag() bool { + return *rs.ref == 0 +} diff --git a/fmt.go b/fmt.go index aed5a52..dd8ac7c 100644 --- a/fmt.go +++ b/fmt.go @@ -4,8 +4,10 @@ import ( "bytes" "fmt" "io" + "os" "reflect" "regexp" + "strconv" "strings" "text/template" "time" @@ -319,6 +321,13 @@ func (f *basefmt) Summary() { } fmt.Fprintln(f.out, elapsed) + // prints used randomization seed + seed, err := strconv.ParseInt(os.Getenv("GODOG_SEED"), 10, 64) + if err == nil && seed != 0 { + fmt.Fprintln(f.out, "") + fmt.Fprintln(f.out, "Randomized with seed:", colors.Yellow(seed)) + } + if text := f.snippets(); text != "" { fmt.Fprintln(f.out, yellow("\nYou can implement step definitions for undefined steps with these snippets:")) fmt.Fprintln(f.out, yellow(text)) diff --git a/options.go b/options.go index b5899d9..36710c2 100644 --- a/options.go +++ b/options.go @@ -2,9 +2,6 @@ package godog import ( "io" - "math/rand" - "strconv" - "time" ) // Options are suite run options @@ -33,7 +30,7 @@ type Options struct { // Any other value will be used as the random seed for shuffling. Re-using the // same seed will allow you to reproduce the shuffle order of a previous run // to isolate an error condition. - Randomize randomSeed + Randomize int64 // Stops on the first failure StopOnFailure bool @@ -57,40 +54,3 @@ type Options struct { // Where it should print formatter output Output io.Writer } - -// randomSeed implements `flag.Value`, see https://golang.org/pkg/flag/#Value -type randomSeed int64 - -// Choose randomly assigns a convenient pseudo-random seed value. -// The resulting seed will be between `1-99999` for later ease of specification. -func (rs *randomSeed) Choose() { - r := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) - *rs = randomSeed(r.Int63n(99998) + 1) -} - -func (rs *randomSeed) Set(s string) error { - if s == "true" { - rs.Choose() - return nil - } - - if s == "false" { - *rs = 0 - return nil - } - - i, err := strconv.ParseInt(s, 10, 64) - *rs = randomSeed(i) - return err -} - -func (rs randomSeed) String() string { - return strconv.FormatInt(int64(rs), 10) -} - -// If a Value has an IsBoolFlag() bool method returning true, the command-line -// parser makes -name equivalent to -name=true rather than using the next -// command-line argument. -func (rs *randomSeed) IsBoolFlag() bool { - return *rs == 0 -} diff --git a/run.go b/run.go index 0aca7f8..9761ecb 100644 --- a/run.go +++ b/run.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "strconv" "github.com/DATA-DOG/godog/colors" ) @@ -109,20 +110,17 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt features, err := parseFeatures(opt.Tags, opt.Paths) fatal(err) - // handle convenience condition where user specified `-1` in Options struct, - // asking us to choose the random value seed for them. - if opt.Randomize == -1 { - opt.Randomize.Choose() - } - r := runner{ fmt: formatter(suite, output), initializer: contextInitializer, features: features, - randomSeed: int64(opt.Randomize), + randomSeed: opt.Randomize, stopOnFailure: opt.StopOnFailure, } + // store chosen seed in environment, so it could be seen in formatter summary report + os.Setenv("GODOG_SEED", strconv.FormatInt(r.randomSeed, 10)) + var failed bool if opt.Concurrency > 1 { failed = r.concurrent(opt.Concurrency)