From 92fbee719c9593813b24a6ef042bcad40324ede4 Mon Sep 17 00:00:00 2001 From: gedi Date: Fri, 5 May 2017 11:03:19 +0300 Subject: [PATCH] tests suite execution function and related errors --- examples/api/version.feature | 2 +- godog.go | 2 +- run.go | 40 +++++++++--- run_test.go | 122 +++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 10 deletions(-) diff --git a/examples/api/version.feature b/examples/api/version.feature index 6a3581a..c09d5ff 100644 --- a/examples/api/version.feature +++ b/examples/api/version.feature @@ -20,6 +20,6 @@ Feature: get version And the response should match json: """ { - "version": "v0.7.1" + "version": "v0.7.2" } """ diff --git a/godog.go b/godog.go index 5e080fc..1cc603b 100644 --- a/godog.go +++ b/godog.go @@ -39,4 +39,4 @@ Godog was inspired by Behat and Cucumber the above description is taken from it' package godog // Version of package - based on Semantic Versioning 2.0.0 http://semver.org/ -const Version = "v0.7.1" +const Version = "v0.7.2" diff --git a/run.go b/run.go index 2fb9c21..01448fb 100644 --- a/run.go +++ b/run.go @@ -10,6 +10,12 @@ import ( "github.com/DATA-DOG/godog/colors" ) +const ( + exitSuccess int = iota + exitFailure + exitOptionError +) + type initializer func(*Suite) type runner struct { @@ -78,6 +84,15 @@ func (r *runner) run() bool { // This method is useful in case if you run // godog in for example TestMain function together // with go tests +// +// The exit codes may vary from: +// 0 - success +// 1 - failed +// 2 - command line usage error +// 128 - or higher, os signal related error exit codes +// +// If there are flag related errors they will +// be directed to os.Stderr func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Options) int { var output io.Writer = os.Stdout if nil != opt.Output { @@ -94,7 +109,7 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt s := &Suite{} contextInitializer(s) s.printStepDefinitions(output) - return 2 // showing help or printing definitions, results exit code - 2 + return exitOptionError } if len(opt.Paths) == 0 { @@ -106,7 +121,7 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt if opt.Concurrency > 1 && !supportsConcurrency(opt.Format) { fmt.Fprintln(os.Stderr, fmt.Errorf("format \"%s\" does not support concurrent execution", opt.Format)) - return 1 + return exitOptionError } formatter := findFmt(opt.Format) if nil == formatter { @@ -119,13 +134,13 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt opt.Format, strings.Join(names, ", "), )) - return 1 + return exitOptionError } features, err := parseFeatures(opt.Tags, opt.Paths) if err != nil { fmt.Fprintln(os.Stderr, err) - return 1 + return exitOptionError } r := runner{ @@ -147,9 +162,9 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt failed = r.run() } if failed && opt.Format != "events" { - return 1 + return exitFailure } - return 0 + return exitSuccess } // Run creates and runs the feature suite. @@ -164,13 +179,22 @@ func RunWithOptions(suite string, contextInitializer func(suite *Suite), opt Opt // // contextInitializer must be able to register // the step definitions and event handlers. +// +// The exit codes may vary from: +// 0 - success +// 1 - failed +// 2 - command line usage error +// 128 - or higher, os signal related error exit codes +// +// If there are flag related errors they will +// be directed to os.Stderr func Run(suite string, contextInitializer func(suite *Suite)) int { var opt Options opt.Output = colors.Colored(os.Stdout) flagSet := FlagSet(&opt) if err := flagSet.Parse(os.Args[1:]); err != nil { fmt.Fprintln(os.Stderr, err) - return 1 + return exitOptionError } opt.Paths = flagSet.Args() @@ -188,5 +212,5 @@ func supportsConcurrency(format string) bool { return true // supports concurrency } - return true // all custom formatters are treated as supporting concurrency + return false // does not support concurrency } diff --git a/run_test.go b/run_test.go index 6149a3d..f30a418 100644 --- a/run_test.go +++ b/run_test.go @@ -3,7 +3,9 @@ package godog import ( "bytes" "fmt" + "io" "io/ioutil" + "os" "strings" "testing" @@ -123,3 +125,123 @@ func TestShouldFailOnError(t *testing.T) { t.Fatal("the suite should have failed") } } + +func TestFailsWithConcurrencyOptionError(t *testing.T) { + stderr, closer := bufErrorPipe(t) + defer closer() + defer stderr.Close() + + opt := Options{ + Format: "pretty", + Paths: []string{"features/load:6"}, + Concurrency: 2, + Output: ioutil.Discard, + } + + status := RunWithOptions("fails", func(_ *Suite) {}, opt) + if status != exitOptionError { + t.Fatalf("expected exit status to be 2, but was: %d", status) + } + closer() + + b, err := ioutil.ReadAll(stderr) + if err != nil { + t.Fatal(err) + } + + out := strings.TrimSpace(string(b)) + if out != `format "pretty" does not support concurrent execution` { + t.Fatalf("unexpected error output: \"%s\"", out) + } +} + +func TestFailsWithUnknownFormatterOptionError(t *testing.T) { + stderr, closer := bufErrorPipe(t) + defer closer() + defer stderr.Close() + + opt := Options{ + Format: "unknown", + Paths: []string{"features/load:6"}, + Output: ioutil.Discard, + } + + status := RunWithOptions("fails", func(_ *Suite) {}, opt) + if status != exitOptionError { + t.Fatalf("expected exit status to be 2, but was: %d", status) + } + closer() + + b, err := ioutil.ReadAll(stderr) + if err != nil { + t.Fatal(err) + } + + out := strings.TrimSpace(string(b)) + if strings.Index(out, `unregistered formatter name: "unknown", use one of`) == -1 { + t.Fatalf("unexpected error output: \"%s\"", out) + } +} + +func TestFailsWithOptionErrorWhenLookingForFeaturesInUnavailablePath(t *testing.T) { + stderr, closer := bufErrorPipe(t) + defer closer() + defer stderr.Close() + + opt := Options{ + Format: "progress", + Paths: []string{"unavailable"}, + Output: ioutil.Discard, + } + + status := RunWithOptions("fails", func(_ *Suite) {}, opt) + if status != exitOptionError { + t.Fatalf("expected exit status to be 2, but was: %d", status) + } + closer() + + b, err := ioutil.ReadAll(stderr) + if err != nil { + t.Fatal(err) + } + + out := strings.TrimSpace(string(b)) + if out != `feature path "unavailable" is not available` { + t.Fatalf("unexpected error output: \"%s\"", out) + } +} + +func TestByDefaultRunsFeaturesPath(t *testing.T) { + opt := Options{ + Format: "progress", + Output: ioutil.Discard, + Strict: true, + } + + status := RunWithOptions("fails", func(_ *Suite) {}, opt) + // should fail in strict mode due to undefined steps + if status != exitFailure { + t.Fatalf("expected exit status to be 1, but was: %d", status) + } + + opt.Strict = false + status = RunWithOptions("succeeds", func(_ *Suite) {}, opt) + // should succeed in non strict mode due to undefined steps + if status != exitSuccess { + t.Fatalf("expected exit status to be 0, but was: %d", status) + } +} + +func bufErrorPipe(t *testing.T) (io.ReadCloser, func()) { + stderr := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + + os.Stderr = w + return r, func() { + w.Close() + os.Stderr = stderr + } +}