From 2e696381c92d0820b39068d6cad233f176489c73 Mon Sep 17 00:00:00 2001 From: gedi Date: Fri, 19 Jun 2015 11:51:35 +0300 Subject: [PATCH] test suite scenario runner --- cmd/godog/main.go | 4 +- features/run.feature | 80 ++++++++++++++++++- utils_test.go => fmt_test.go | 0 suite_test.go | 148 +++++++++++++++++++++++++---------- 4 files changed, 189 insertions(+), 43 deletions(-) rename utils_test.go => fmt_test.go (100%) diff --git a/cmd/godog/main.go b/cmd/godog/main.go index a7214de..48d5d6d 100644 --- a/cmd/godog/main.go +++ b/cmd/godog/main.go @@ -39,13 +39,13 @@ func main() { // @TODO: support for windows cmd := exec.Command("sh", "-c", c) cmd.Stdout = stdout - // @TODO: do not read stderr on production version cmd.Stderr = stdout err = cmd.Run() switch err.(type) { case *exec.ExitError: - os.Remove(builtFile) + // then there is a suite error, we need to provide a + // way to see the built file and do not remove it here os.Exit(1) case *exec.Error: os.Remove(builtFile) diff --git a/features/run.feature b/features/run.feature index c6ad874..a10a7c0 100644 --- a/features/run.feature +++ b/features/run.feature @@ -14,5 +14,83 @@ Feature: run features Then I should have 1 scenario registered """ When I run feature suite - Then the suite should have passed successfully + Then the suite should have passed + And the following steps should be passed: + """ + a feature path "features/load.feature:6" + I parse features + I should have 1 scenario registered + """ + Scenario: should skip steps after failure + Given a feature "failed.feature" file: + """ + Feature: failed feature + + Scenario: parse a scenario + Given a failing step + When I parse features + Then I should have 1 scenario registered + """ + When I run feature suite + Then the suite should have failed + And the following step should be failed: + """ + a failing step + """ + And the following steps should be skipped: + """ + I parse features + I should have 1 scenario registered + """ + + Scenario: should skip all scenarios if background fails + Given a feature "failed.feature" file: + """ + Feature: failed feature + + Background: + Given a failing step + + Scenario: parse a scenario + Given a feature path "features/load.feature:6" + When I parse features + Then I should have 1 scenario registered + """ + When I run feature suite + Then the suite should have failed + And the following step should be failed: + """ + a failing step + """ + And the following steps should be skipped: + """ + a feature path "features/load.feature:6" + I parse features + I should have 1 scenario registered + """ + + Scenario: should skip steps after undefined + Given a feature "undefined.feature" file: + """ + Feature: undefined feature + + Scenario: parse a scenario + Given a feature path "features/load.feature:6" + When undefined action + Then I should have 1 scenario registered + """ + When I run feature suite + Then the suite should have passed # we do not treat undefined scenarios as fails + And the following step should be passed: + """ + a feature path "features/load.feature:6" + """ + And the following step should be undefined: + """ + undefined action + """ + And the following step should be skipped: + """ + I should have 1 scenario registered + """ diff --git a/utils_test.go b/fmt_test.go similarity index 100% rename from utils_test.go rename to fmt_test.go diff --git a/suite_test.go b/suite_test.go index cdfc654..ef5ebdd 100644 --- a/suite_test.go +++ b/suite_test.go @@ -7,19 +7,44 @@ import ( "github.com/DATA-DOG/godog/gherkin" ) +func SuiteContext(s Suite) { + c := &suiteContext{} + + s.BeforeScenario(c) + + s.Step(`^a feature path "([^"]*)"$`, c.featurePath) + s.Step(`^I parse features$`, c.parseFeatures) + s.Step(`^I'm listening to suite events$`, c.iAmListeningToSuiteEvents) + s.Step(`^I run feature suite$`, c.iRunFeatureSuite) + s.Step(`^a feature "([^"]*)" file:$`, c.aFeatureFile) + s.Step(`^the suite should have (passed|failed)$`, c.theSuiteShouldHave) + + s.Step(`^I should have ([\d]+) features? files?:$`, c.iShouldHaveNumFeatureFiles) + s.Step(`^I should have ([\d]+) scenarios? registered$`, c.numScenariosRegistered) + s.Step(`^there (was|were) ([\d]+) "([^"]*)" events? fired$`, c.thereWereNumEventsFired) + s.Step(`^there was event triggered before scenario "([^"]*)"$`, c.thereWasEventTriggeredBeforeScenario) + s.Step(`^these events had to be fired for a number of times:$`, c.theseEventsHadToBeFiredForNumberOfTimes) + + s.Step(`^a failing step`, c.aFailingStep) + s.Step(`^this step should fail`, c.aFailingStep) + s.Step(`^the following steps? should be (passed|failed|skipped|undefined):`, c.followingStepsShouldHave) +} + type firedEvent struct { name string args []interface{} } -type suiteFeature struct { +type suiteContext struct { testedSuite *suite events []*firedEvent + fmt *testFormatter } -func (s *suiteFeature) HandleBeforeScenario(*gherkin.Scenario) { +func (s *suiteContext) HandleBeforeScenario(*gherkin.Scenario) { // reset whole suite with the state - s.testedSuite = &suite{fmt: &testFormatter{}} + s.fmt = &testFormatter{} + s.testedSuite = &suite{fmt: s.fmt} // our tested suite will have the same context registered SuiteContext(s.testedSuite) // reset feature paths @@ -28,7 +53,65 @@ func (s *suiteFeature) HandleBeforeScenario(*gherkin.Scenario) { s.events = []*firedEvent{} } -func (s *suiteFeature) iAmListeningToSuiteEvents(args ...*Arg) error { +func (s *suiteContext) followingStepsShouldHave(args ...*Arg) error { + var expected []string = args[1].PyString().Lines + var actual, unmatched []string + var matched []int + + switch args[0].String() { + case "passed": + for _, st := range s.fmt.passed { + actual = append(actual, st.step.Text) + } + case "failed": + for _, st := range s.fmt.failed { + actual = append(actual, st.step.Text) + } + case "skipped": + for _, st := range s.fmt.skipped { + actual = append(actual, st.step.Text) + } + case "undefined": + for _, st := range s.fmt.undefined { + actual = append(actual, st.step.Text) + } + default: + return fmt.Errorf("unexpected step status wanted: %s", args[0].String()) + } + + if len(expected) > len(actual) { + return fmt.Errorf("number of expected %s steps: %d is less than actual %s steps: %d", args[0].String(), len(expected), args[0].String(), len(actual)) + } + + for _, a := range actual { + for i, e := range expected { + if a == e { + matched = append(matched, i) + break + } + } + } + + if len(matched) == len(expected) { + return nil + } + + for i, s := range expected { + var found bool + for _, m := range matched { + if i == m { + found = true + } + } + if !found { + unmatched = append(unmatched, s) + } + } + + return fmt.Errorf("the steps: %s - is not %s", strings.Join(unmatched, ", "), args[0].String()) +} + +func (s *suiteContext) iAmListeningToSuiteEvents(args ...*Arg) error { s.testedSuite.BeforeSuite(BeforeSuiteHandlerFunc(func() { s.events = append(s.events, &firedEvent{"BeforeSuite", []interface{}{}}) })) @@ -50,12 +133,12 @@ func (s *suiteFeature) iAmListeningToSuiteEvents(args ...*Arg) error { return nil } -func (s *suiteFeature) aFailingStep(...*Arg) error { +func (s *suiteContext) aFailingStep(...*Arg) error { return fmt.Errorf("intentional failure") } // parse a given feature file body as a feature -func (s *suiteFeature) aFeatureFile(args ...*Arg) error { +func (s *suiteContext) aFeatureFile(args ...*Arg) error { name := args[0].String() body := args[1].PyString().Raw feature, err := gherkin.Parse(strings.NewReader(body), name) @@ -63,24 +146,31 @@ func (s *suiteFeature) aFeatureFile(args ...*Arg) error { return err } -func (s *suiteFeature) featurePath(args ...*Arg) error { +func (s *suiteContext) featurePath(args ...*Arg) error { cfg.paths = append(cfg.paths, args[0].String()) return nil } -func (s *suiteFeature) parseFeatures(args ...*Arg) (err error) { - s.testedSuite.features, err = cfg.features() - return +func (s *suiteContext) parseFeatures(args ...*Arg) error { + features, err := cfg.features() + if err != nil { + return err + } + s.testedSuite.features = append(s.testedSuite.features, features...) + return nil } -func (s *suiteFeature) theSuitePassedSuccessfully(...*Arg) error { - if s.testedSuite.failed { +func (s *suiteContext) theSuiteShouldHave(args ...*Arg) error { + if s.testedSuite.failed && args[0].String() == "passed" { return fmt.Errorf("the feature suite has failed") } + if !s.testedSuite.failed && args[0].String() == "failed" { + return fmt.Errorf("the feature suite has passed") + } return nil } -func (s *suiteFeature) iShouldHaveNumFeatureFiles(args ...*Arg) error { +func (s *suiteContext) iShouldHaveNumFeatureFiles(args ...*Arg) error { if len(s.testedSuite.features) != args[0].Int() { return fmt.Errorf("expected %d features to be parsed, but have %d", args[0].Int(), len(s.testedSuite.features)) } @@ -100,7 +190,7 @@ func (s *suiteFeature) iShouldHaveNumFeatureFiles(args ...*Arg) error { return nil } -func (s *suiteFeature) iRunFeatureSuite(args ...*Arg) error { +func (s *suiteContext) iRunFeatureSuite(args ...*Arg) error { if err := s.parseFeatures(); err != nil { return err } @@ -108,7 +198,7 @@ func (s *suiteFeature) iRunFeatureSuite(args ...*Arg) error { return nil } -func (s *suiteFeature) numScenariosRegistered(args ...*Arg) (err error) { +func (s *suiteContext) numScenariosRegistered(args ...*Arg) (err error) { var num int for _, ft := range s.testedSuite.features { num += len(ft.Scenarios) @@ -119,7 +209,7 @@ func (s *suiteFeature) numScenariosRegistered(args ...*Arg) (err error) { return } -func (s *suiteFeature) thereWereNumEventsFired(args ...*Arg) error { +func (s *suiteContext) thereWereNumEventsFired(args ...*Arg) error { var num int for _, event := range s.events { if event.name == args[2].String() { @@ -132,7 +222,7 @@ func (s *suiteFeature) thereWereNumEventsFired(args ...*Arg) error { return nil } -func (s *suiteFeature) thereWasEventTriggeredBeforeScenario(args ...*Arg) error { +func (s *suiteContext) thereWasEventTriggeredBeforeScenario(args ...*Arg) error { var found []string for _, event := range s.events { if event.name != "BeforeScenario" { @@ -154,7 +244,7 @@ func (s *suiteFeature) thereWasEventTriggeredBeforeScenario(args ...*Arg) error return fmt.Errorf(`expected "%s" scenario, but got these fired %s`, args[0].String(), `"`+strings.Join(found, `", "`)+`"`) } -func (s *suiteFeature) theseEventsHadToBeFiredForNumberOfTimes(args ...*Arg) error { +func (s *suiteContext) theseEventsHadToBeFiredForNumberOfTimes(args ...*Arg) error { tbl := args[0].Table() if len(tbl.Rows[0]) != 2 { return fmt.Errorf("expected two columns for event table row, got: %d", len(tbl.Rows[0])) @@ -172,25 +262,3 @@ func (s *suiteFeature) theseEventsHadToBeFiredForNumberOfTimes(args ...*Arg) err } return nil } - -func SuiteContext(g Suite) { - s := &suiteFeature{} - - g.BeforeScenario(s) - - g.Step(`^a feature path "([^"]*)"$`, s.featurePath) - g.Step(`^I parse features$`, s.parseFeatures) - g.Step(`^I'm listening to suite events$`, s.iAmListeningToSuiteEvents) - g.Step(`^I run feature suite$`, s.iRunFeatureSuite) - g.Step(`^a feature "([^"]*)" file:$`, s.aFeatureFile) - g.Step(`^the suite should have passed successfully$`, s.theSuitePassedSuccessfully) - - g.Step(`^I should have ([\d]+) features? files?:$`, s.iShouldHaveNumFeatureFiles) - g.Step(`^I should have ([\d]+) scenarios? registered$`, s.numScenariosRegistered) - g.Step(`^there (was|were) ([\d]+) "([^"]*)" events? fired$`, s.thereWereNumEventsFired) - g.Step(`^there was event triggered before scenario "([^"]*)"$`, s.thereWasEventTriggeredBeforeScenario) - g.Step(`^these events had to be fired for a number of times:$`, s.theseEventsHadToBeFiredForNumberOfTimes) - - g.Step(`^a failing step`, s.aFailingStep) - g.Step(`^this step should fail`, s.aFailingStep) -}