diff --git a/features/formatter/events.feature b/features/formatter/events.feature index d9c2b2a..cd99b33 100644 --- a/features/formatter/events.feature +++ b/features/formatter/events.feature @@ -14,7 +14,7 @@ Feature: event stream formatter """ Scenario: should process simple scenario - Given a feature path "features/load.feature:21" + Given a feature path "features/load.feature:22" When I run feature suite with formatter "events" Then the following events should be fired: """ @@ -35,7 +35,7 @@ Feature: event stream formatter """ Scenario: should process outline scenario - Given a feature path "features/load.feature:29" + Given a feature path "features/load.feature:30" When I run feature suite with formatter "events" Then the following events should be fired: """ diff --git a/features/lang.feature b/features/lang.feature index 0cf35a0..829a939 100644 --- a/features/lang.feature +++ b/features/lang.feature @@ -8,10 +8,11 @@ Savybė: užkrauti savybes Scenarijus: savybių užkrovimas iš aplanko Duota savybių aplankas "features" Kai aš išskaitau savybes - Tada aš turėčiau turėti 8 savybių failus: + Tada aš turėčiau turėti 9 savybių failus: """ features/background.feature features/events.feature + features/formatter/cucumber.feature features/formatter/events.feature features/lang.feature features/load.feature diff --git a/features/load.feature b/features/load.feature index f15cee1..22ed683 100644 --- a/features/load.feature +++ b/features/load.feature @@ -6,10 +6,11 @@ Feature: load features Scenario: load features within path Given a feature path "features" When I parse features - Then I should have 8 feature files: + Then I should have 9 feature files: """ features/background.feature features/events.feature + features/formatter/cucumber.feature features/formatter/events.feature features/lang.feature features/load.feature diff --git a/fmt_cucumber.go b/fmt_cucumber.go index ff3d131..0824d34 100644 --- a/fmt_cucumber.go +++ b/fmt_cucumber.go @@ -1,12 +1,22 @@ package godog +/* + The specification for the formatting originated from https://www.relishapp.com/cucumber/cucumber/docs/formatters/json-output-formatter. + I found that documentation was misleading or out dated. To validate formatting I create a ruby cucumber test harness and ran the + same feature files through godog and the ruby cucumber. + + The docstrings in the cucumber.feature represent the cucumber output for those same feature definitions. + + I did note that comments in ruby could be at just about any level in particular Feature, Scenario and Step. In godog I + could only find comments under the Feature data structure. + */ import ( "fmt" "io" "time" "strings" - "github.com/DATA-DOG/godog/gherkin" "encoding/json" + "github.com/DATA-DOG/godog/gherkin" "strconv" ) @@ -28,24 +38,32 @@ func cucumberFunc(suite string, out io.Writer) Formatter { return formatter } -// Replace spaces with - +// Replace spaces with - This function is used to create the "id" fields of the cucumber output. func makeId(name string) string { return strings.Replace(strings.ToLower(name), " ", "-", -1) } +// The sequence of type structs are used to marshall the json object. type cukeComment struct { Value string `json:"value"` - Line int `json:"line"` + Line int `json:"line"` } + +type cukeDocstring struct { + Value string `json:"value"` + ContentType string `json:"content_type"` + Line int `json:"line"` +} + type cukeTag struct { Name string `json:"name"` - Line int `json:"line"` + Line int `json:"line"` } type cukeResult struct { Status string `json:"status"` Error string `json:"error_message,omitempty"` - Duration int `json:"duration"` + Duration int `json:"duration"` } type cukeMatch struct { @@ -53,57 +71,59 @@ type cukeMatch struct { } type cukeStep struct { - Keyword string `json:"keyword"` - Name string `json:"name"` - Line int `json:"line"` - Match cukeMatch `json:"match"` + Keyword string `json:"keyword"` + Name string `json:"name"` + Line int `json:"line"` + Docstring cukeDocstring `json:"doc_string,omitempty"` + Match cukeMatch `json:"match"` Result cukeResult `json:"result"` } type cukeElement struct { - Id string `json:"id"` - Keyword string `json:"keyword"` - Name string `json:"name"` - Description string `json:"description"` - Line int `json:"line"` - Type string `json:"type"` - Tags []cukeTag `json:"tags,omitempty"` - Steps []cukeStep `json:"steps"` + Id string `json:"id"` + Keyword string `json:"keyword"` + Name string `json:"name"` + Description string `json:"description"` + Line int `json:"line"` + Type string `json:"type"` + Tags []cukeTag `json:"tags,omitempty"` + Steps []cukeStep `json:"steps,omitempty"` } type cukeFeatureJson struct { - Uri string `json:"uri"` - Id string `json:"id"` - Keyword string `json:"keyword"` - Name string `json:"name"` - Description string `json:"description"` - Line int `json:"line"` - Comments []cukeComment `json:"comments,omitempty"` - Tags []cukeTag `json:"tags,omitempty"` - Elements []cukeElement `json:"elements"` + Uri string `json:"uri"` + Id string `json:"id"` + Keyword string `json:"keyword"` + Name string `json:"name"` + Description string `json:"description"` + Line int `json:"line"` + Comments []cukeComment `json:"comments,omitempty"` + Tags []cukeTag `json:"tags,omitempty"` + Elements []cukeElement `json:"elements,omitempty"` } type cukefmt struct { basefmt - // currently running feature path, to be part of id. - // this is sadly not passed by gherkin nodes. - // it restricts this formatter to run only in synchronous single - // threaded execution. Unless running a copy of formatter for each feature - path string - stat stepType // last step status, before skipped - outlineSteps int // number of current outline scenario steps - id string // current test id. - results []cukeFeatureJson // structure that represent cuke results - curStep *cukeStep // track the current step - curElement *cukeElement // track the current element - curFeature *cukeFeatureJson // track the current feature - curOutline cukeElement // Each example show up as an outline element but the outline is parsed only once - // so I need to keep track of the current outline - curRow int // current row of the example table as it is being processed. - curExampleTags []cukeTag // temporary storage for tags associate with the current example table. - startTime time.Time - curExampleName string + // currently running feature path, to be part of id. + // this is sadly not passed by gherkin nodes. + // it restricts this formatter to run only in synchronous single + // threaded execution. Unless running a copy of formatter for each feature + path string + stat stepType // last step status, before skipped + outlineSteps int // number of current outline scenario steps + id string // current test id. + results []cukeFeatureJson // structure that represent cuke results + curStep *cukeStep // track the current step + curElement *cukeElement // track the current element + curFeature *cukeFeatureJson // track the current feature + curOutline cukeElement // Each example show up as an outline element but the outline is parsed only once + // so I need to keep track of the current outline + curRow int // current row of the example table as it is being processed. + curExampleTags []cukeTag // temporary storage for tags associate with the current example table. + startTime time.Time // used to time duration of the step execution + curExampleName string // Due to the fact that examples are parsed once and then iterated over for each result then we need to keep track + // of the example name inorder to build id fields. } func (f *cukefmt) Node(n interface{}) { @@ -114,6 +134,7 @@ func (f *cukefmt) Node(n interface{}) { // When the example definition is seen we just need track the id and // append the name associated with the example as part of the id. case *gherkin.Examples: + f.curExampleName = makeId(t.Name) f.curRow = 2 // there can be more than one example set per outline so reset row count. // cucumber counts the header row as an example when creating the id. @@ -135,23 +156,23 @@ func (f *cukefmt) Node(n interface{}) { f.curOutline.Keyword = t.Keyword f.curOutline.Id = f.curFeature.Id + ";" + makeId(t.Name) f.curOutline.Type = "scenario" - f.curOutline.Tags = make([]cukeTag, len(t.Tags) + len(f.curFeature.Tags)) + f.curOutline.Tags = make([]cukeTag, len(t.Tags)+len(f.curFeature.Tags)) // apply feature level tags - if (len(f.curOutline.Tags) > 0) { + if len(f.curOutline.Tags) > 0 { copy(f.curOutline.Tags, f.curFeature.Tags) // apply outline level tags. for idx, element := range t.Tags { - f.curOutline.Tags[idx + len(f.curFeature.Tags)].Line = element.Location.Line - f.curOutline.Tags[idx + len(f.curFeature.Tags)].Name = element.Name + f.curOutline.Tags[idx+len(f.curFeature.Tags)].Line = element.Location.Line + f.curOutline.Tags[idx+len(f.curFeature.Tags)].Name = element.Name } } // This scenario adds the element to the output immediately. case *gherkin.Scenario: f.curFeature.Elements = append(f.curFeature.Elements, cukeElement{}) - f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements) - 1] + f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements)-1] f.curElement.Name = t.Name f.curElement.Line = t.Location.Line @@ -159,16 +180,16 @@ func (f *cukefmt) Node(n interface{}) { f.curElement.Keyword = t.Keyword f.curElement.Id = f.curFeature.Id + ";" + makeId(t.Name) f.curElement.Type = "scenario" - f.curElement.Tags = make([]cukeTag, len(t.Tags) + len(f.curFeature.Tags)) + f.curElement.Tags = make([]cukeTag, len(t.Tags)+len(f.curFeature.Tags)) - if (len(f.curElement.Tags) > 0) { + if len(f.curElement.Tags) > 0 { // apply feature level tags copy(f.curElement.Tags, f.curFeature.Tags) // apply scenario level tags. for idx, element := range t.Tags { - f.curElement.Tags[idx + len(f.curFeature.Tags)].Line = element.Location.Line - f.curElement.Tags[idx + len(f.curFeature.Tags)].Name = element.Name + f.curElement.Tags[idx+len(f.curFeature.Tags)].Line = element.Location.Line + f.curElement.Tags[idx+len(f.curFeature.Tags)].Name = element.Name } } @@ -181,7 +202,7 @@ func (f *cukefmt) Node(n interface{}) { tmpElem.Id = tmpElem.Id + ";" + f.curExampleName + ";" + strconv.Itoa(f.curRow) f.curRow++ f.curFeature.Elements = append(f.curFeature.Elements, tmpElem) - f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements) - 1] + f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements)-1] // copy in example level tags. f.curElement.Tags = append(f.curElement.Tags, f.curExampleTags...) @@ -197,7 +218,7 @@ func (f *cukefmt) Feature(ft *gherkin.Feature, p string, c []byte) { f.id = makeId(ft.Name) f.results = append(f.results, cukeFeatureJson{}) - f.curFeature = &f.results[len(f.results) - 1] + f.curFeature = &f.results[len(f.results)-1] f.curFeature.Uri = p f.curFeature.Name = ft.Name f.curFeature.Keyword = ft.Keyword @@ -213,7 +234,7 @@ func (f *cukefmt) Feature(ft *gherkin.Feature, p string, c []byte) { f.curFeature.Comments = make([]cukeComment, len(ft.Comments)) for idx, comment := range ft.Comments { - f.curFeature.Comments[idx].Value = comment.Text + f.curFeature.Comments[idx].Value = strings.TrimSpace(comment.Text) f.curFeature.Comments[idx].Line = comment.Location.Line } @@ -251,12 +272,18 @@ func (f *cukefmt) Defined(step *gherkin.Step, def *StepDef) { f.startTime = time.Now() // start timing the step f.curElement.Steps = append(f.curElement.Steps, cukeStep{}) - f.curStep = &f.curElement.Steps[len(f.curElement.Steps) - 1] + f.curStep = &f.curElement.Steps[len(f.curElement.Steps)-1] f.curStep.Name = step.Text f.curStep.Line = step.Location.Line f.curStep.Keyword = step.Keyword + if _, ok := step.Argument.(*gherkin.DocString); ok { + f.curStep.Docstring.ContentType = strings.TrimSpace(step.Argument.(*gherkin.DocString).ContentType) + f.curStep.Docstring.Line = step.Argument.(*gherkin.DocString).Location.Line + f.curStep.Docstring.Value = step.Argument.(*gherkin.DocString).Content + } + if def != nil { f.curStep.Match.Location = strings.Split(def.definitionID(), " ")[0] } @@ -265,28 +292,28 @@ func (f *cukefmt) Defined(step *gherkin.Step, def *StepDef) { func (f *cukefmt) Passed(step *gherkin.Step, match *StepDef) { f.basefmt.Passed(step, match) f.stat = passed - f.step(f.passed[len(f.passed) - 1]) + f.step(f.passed[len(f.passed)-1]) } func (f *cukefmt) Skipped(step *gherkin.Step) { f.basefmt.Skipped(step) - f.step(f.skipped[len(f.skipped) - 1]) + f.step(f.skipped[len(f.skipped)-1]) } func (f *cukefmt) Undefined(step *gherkin.Step) { f.basefmt.Undefined(step) f.stat = undefined - f.step(f.undefined[len(f.undefined) - 1]) + f.step(f.undefined[len(f.undefined)-1]) } func (f *cukefmt) Failed(step *gherkin.Step, match *StepDef, err error) { f.basefmt.Failed(step, match, err) f.stat = failed - f.step(f.failed[len(f.failed) - 1]) + f.step(f.failed[len(f.failed)-1]) } func (f *cukefmt) Pending(step *gherkin.Step, match *StepDef) { f.stat = pending f.basefmt.Pending(step, match) - f.step(f.pending[len(f.pending) - 1]) + f.step(f.pending[len(f.pending)-1]) } diff --git a/run.go b/run.go index eea6b6a..cb73447 100644 --- a/run.go +++ b/run.go @@ -157,6 +157,9 @@ func supportsConcurrency(format string) bool { return false case "pretty": return false + case "cucumber": + return false } + return true // all custom formatters are treated as supporting concurrency } diff --git a/suite_test.go b/suite_test.go index 638a0c6..36e1d7f 100644 --- a/suite_test.go +++ b/suite_test.go @@ -4,13 +4,13 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/DATA-DOG/godog/gherkin" "os" "path/filepath" + "reflect" "strconv" "strings" "testing" - - "github.com/DATA-DOG/godog/gherkin" ) func TestMain(m *testing.M) { @@ -66,6 +66,10 @@ func SuiteContext(s *Suite) { s.Step(`^passing step$`, func() error { return nil }) + + // Introduced to test formatter/cucumber.feature + s.Step(`^the rendered json will be as follows:$`, c.theRenderJsonWillBe) + } type firedEvent struct { @@ -185,7 +189,7 @@ func (s *suiteContext) followingStepsShouldHave(status string, steps *gherkin.Do } if len(expected) > len(actual) { - return fmt.Errorf("number of expected %s steps: %d is less than actual %s steps: %d", status, len(expected), status, len(actual)) + return fmt.Errorf("number of expeted %s steps: %d is less than actual %s steps: %d", status, len(expected), status, len(actual)) } for _, a := range actual { @@ -376,3 +380,103 @@ func (s *suiteContext) theseEventsHadToBeFiredForNumberOfTimes(tbl *gherkin.Data } return nil } + +func (s *suiteContext) theRenderJsonWillBe2(docstring *gherkin.DocString) error { + + var expected bytes.Buffer + var actual bytes.Buffer + + if err := json.Compact(&expected, []byte(docstring.Content)); err != nil { + return err + } + + if err := json.Compact(&actual, s.out.Bytes()); err != nil { + return err + } + + if string(expected.Bytes()) != string(actual.Bytes()) { + return fmt.Errorf("format mismatch expected:[%v] actual:[%v]", string(expected.Bytes()), string(actual.Bytes())) + } + + return nil +} + +func (s *suiteContext) theRenderJsonWillBe(docstring *gherkin.DocString) error { + + var expected interface{} + if err := json.Unmarshal([]byte(docstring.Content), &expected); err != nil { + return err + } + + var actual interface{} + if err := json.Unmarshal(s.out.Bytes(), &actual); err != nil { + return err + } + + expectedArr := expected.([]interface{}) + actualArr := actual.([]interface{}) + + for idx, entry := range expectedArr { + if err := s.mapCompare(entry.(map[string]interface{}), actualArr[idx].(map[string]interface{})); err != nil { + return err + } + } + return nil +} + +/* + Due to specialize matching logic to ignore exact matches on the "location" and "duration" fields. It was + necessary to create this compare function to validate the values of the map. +*/ +func (s *suiteContext) mapCompare(expected map[string]interface{}, actual map[string]interface{}) error { + + // Process all keys in the map and handle them based on the type of the field. + for k, v := range expected { + + if actual[k] == nil { + return fmt.Errorf("No matching field in actual:[%s] expected value:[%v]",k,v) + } + // Process other maps via recursion + if reflect.TypeOf(v).Kind() == reflect.Map { + if err := s.mapCompare(v.(map[string]interface{}), actual[k].(map[string]interface{})); err != nil { + return err + } + // This is an array of maps show as a slice + } else if reflect.TypeOf(v).Kind() == reflect.Slice { + for i, e := range v.([]interface{}) { + if err := s.mapCompare(e.(map[string]interface{}), actual[k].([]interface{})[i].(map[string]interface{})); err != nil { + return err + } + } + // We need special rules to check location so that we are not bound to version of the code. + } else if k == "location" { + if !strings.Contains(actual[k].(string), "suite_test.go:") { + return fmt.Errorf("location has unexpected filename [%s] should contains suite_test.go", + actual[k]) + } + // We need special rules to validate duration too. + } else if k == "duration" { + if actual[k].(float64) <= 0 { + return fmt.Errorf("duration is <= zero: actual:[%v]", actual[k]) + } + // default numbers in json are coming as float64 + } else if reflect.TypeOf(v).Kind() == reflect.Float64 { + if v.(float64) != actual[k].(float64) { + if v.(float64) != actual[k].(float64) { + return fmt.Errorf("Field:[%s] not matching expected:[%v] actual:[%v]", + k, v, actual[k]) + } + } + + } else if reflect.TypeOf(v).Kind() == reflect.String { + if v.(string) != actual[k].(string) { + return fmt.Errorf("Field:[%s] not matching expected:[%v] actual:[%v]", + k, v, actual[k]) + } + } else { + return fmt.Errorf("Unexepcted type encountered in json at key:[%s] Type:[%v]", k, reflect.TypeOf(v).Kind()) + } + } + + return nil +}