Discovered defects resulting from not comparing the expected and actual structures. Times when actual contained more fields than expected.

fmt_cucumber.go

@@ -64,7 +64,7 @@ type cukeTag struct {
 type cukeResult struct {
 	Status   string `json:"status"`
 	Error    string `json:"error_message,omitempty"`
-	Duration int    `json:"duration"`
+	Duration *int    `json:"duration,omitempty"`
 }
 
 type cukeMatch struct {
@@ -75,7 +75,7 @@ type cukeStep struct {
 	Keyword   string        `json:"keyword"`
 	Name      string        `json:"name"`
 	Line      int           `json:"line"`
-	Docstring cukeDocstring `json:"doc_string,omitempty"`
+	Docstring *cukeDocstring `json:"doc_string,omitempty"`
 	Match     cukeMatch     `json:"match"`
 	Result    cukeResult    `json:"result"`
 }
@@ -157,7 +157,7 @@ func (f *cukefmt) Node(n interface{}) {
 		f.curOutline.Keyword = t.Keyword
 		f.curOutline.ID = f.curFeature.ID + ";" + makeID(t.Name)
 		f.curOutline.Type = "scenario"
-		f.curOutline.Tags = make([]cukeTag, len(t.Tags)+len(f.curFeature.Tags))
+		f.curOutline.Tags = make([]cukeTag, len(t.Tags) + len(f.curFeature.Tags))
 
 		// apply feature level tags
 		if len(f.curOutline.Tags) > 0 {
@@ -165,15 +165,15 @@ func (f *cukefmt) Node(n interface{}) {
 
 			// apply outline level tags.
 			for idx, element := range t.Tags {
-				f.curOutline.Tags[idx+len(f.curFeature.Tags)].Line = element.Location.Line
+				f.curOutline.Tags[idx + len(f.curFeature.Tags)].Line = element.Location.Line
-				f.curOutline.Tags[idx+len(f.curFeature.Tags)].Name = element.Name
+				f.curOutline.Tags[idx + len(f.curFeature.Tags)].Name = element.Name
 			}
 		}
 
 	// This scenario adds the element to the output immediately.
 	case *gherkin.Scenario:
 		f.curFeature.Elements = append(f.curFeature.Elements, cukeElement{})
-		f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements)-1]
+		f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements) - 1]
 
 		f.curElement.Name = t.Name
 		f.curElement.Line = t.Location.Line
@@ -181,7 +181,7 @@ func (f *cukefmt) Node(n interface{}) {
 		f.curElement.Keyword = t.Keyword
 		f.curElement.ID = f.curFeature.ID + ";" + makeID(t.Name)
 		f.curElement.Type = "scenario"
-		f.curElement.Tags = make([]cukeTag, len(t.Tags)+len(f.curFeature.Tags))
+		f.curElement.Tags = make([]cukeTag, len(t.Tags) + len(f.curFeature.Tags))
 
 		if len(f.curElement.Tags) > 0 {
 			// apply feature level tags
@@ -189,8 +189,8 @@ func (f *cukefmt) Node(n interface{}) {
 
 			// apply scenario level tags.
 			for idx, element := range t.Tags {
-				f.curElement.Tags[idx+len(f.curFeature.Tags)].Line = element.Location.Line
+				f.curElement.Tags[idx + len(f.curFeature.Tags)].Line = element.Location.Line
-				f.curElement.Tags[idx+len(f.curFeature.Tags)].Name = element.Name
+				f.curElement.Tags[idx + len(f.curFeature.Tags)].Name = element.Name
 			}
 		}
 
@@ -202,7 +202,7 @@ func (f *cukefmt) Node(n interface{}) {
 		tmpElem.ID = tmpElem.ID + ";" + f.curExampleName + ";" + strconv.Itoa(f.curRow)
 		f.curRow++
 		f.curFeature.Elements = append(f.curFeature.Elements, tmpElem)
-		f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements)-1]
+		f.curElement = &f.curFeature.Elements[len(f.curFeature.Elements) - 1]
 
 		// copy in example level tags.
 		f.curElement.Tags = append(f.curElement.Tags, f.curExampleTags...)
@@ -218,7 +218,7 @@ func (f *cukefmt) Feature(ft *gherkin.Feature, p string, c []byte) {
 	f.ID = makeID(ft.Name)
 	f.results = append(f.results, cukeFeatureJSON{})
 
-	f.curFeature = &f.results[len(f.results)-1]
+	f.curFeature = &f.results[len(f.results) - 1]
 	f.curFeature.URI = p
 	f.curFeature.Name = ft.Name
 	f.curFeature.Keyword = ft.Keyword
@@ -250,17 +250,20 @@ func (f *cukefmt) Summary() {
 
 func (f *cukefmt) step(res *stepResult) {
 
+
 	// determine if test case has finished
 	switch t := f.owner.(type) {
 	case *gherkin.TableRow:
-		f.curStep.Result.Duration = int(time.Since(f.startTime).Nanoseconds())
+		d := int(time.Since(f.startTime).Nanoseconds())
+		f.curStep.Result.Duration = &d
 		f.curStep.Line = t.Location.Line
 		f.curStep.Result.Status = res.typ.String()
 		if res.err != nil {
 			f.curStep.Result.Error = res.err.Error()
 		}
 	case *gherkin.Scenario:
-		f.curStep.Result.Duration = int(time.Since(f.startTime).Nanoseconds())
+		d := int(time.Since(f.startTime).Nanoseconds())
+		f.curStep.Result.Duration = &d
 		f.curStep.Result.Status = res.typ.String()
 		if res.err != nil {
 			f.curStep.Result.Error = res.err.Error()
@@ -272,13 +275,14 @@ func (f *cukefmt) Defined(step *gherkin.Step, def *StepDef) {
 
 	f.startTime = time.Now() // start timing the step
 	f.curElement.Steps = append(f.curElement.Steps, cukeStep{})
-	f.curStep = &f.curElement.Steps[len(f.curElement.Steps)-1]
+	f.curStep = &f.curElement.Steps[len(f.curElement.Steps) - 1]
 
 	f.curStep.Name = step.Text
 	f.curStep.Line = step.Location.Line
 	f.curStep.Keyword = step.Keyword
 
 	if _, ok := step.Argument.(*gherkin.DocString); ok {
+		f.curStep.Docstring = &cukeDocstring{}
 		f.curStep.Docstring.ContentType = strings.TrimSpace(step.Argument.(*gherkin.DocString).ContentType)
 		f.curStep.Docstring.Line = step.Argument.(*gherkin.DocString).Location.Line
 		f.curStep.Docstring.Value = step.Argument.(*gherkin.DocString).Content
@@ -292,28 +296,39 @@ func (f *cukefmt) Defined(step *gherkin.Step, def *StepDef) {
 func (f *cukefmt) Passed(step *gherkin.Step, match *StepDef) {
 	f.basefmt.Passed(step, match)
 	f.stat = passed
-	f.step(f.passed[len(f.passed)-1])
+	f.step(f.passed[len(f.passed) - 1])
 }
 
 func (f *cukefmt) Skipped(step *gherkin.Step) {
 	f.basefmt.Skipped(step)
-	f.step(f.skipped[len(f.skipped)-1])
+	f.step(f.skipped[len(f.skipped) - 1])
+
+	// no duration reported for skipped.
+	f.curStep.Result.Duration = nil
 }
 
 func (f *cukefmt) Undefined(step *gherkin.Step) {
 	f.basefmt.Undefined(step)
 	f.stat = undefined
-	f.step(f.undefined[len(f.undefined)-1])
+	f.step(f.undefined[len(f.undefined) - 1])
+
+	// the location for undefined is the feature file location not the step file.
+	f.curStep.Match.Location = fmt.Sprintf("%s:%d",f.path,step.Location.Line)
+	f.curStep.Result.Duration = nil
 }
 
 func (f *cukefmt) Failed(step *gherkin.Step, match *StepDef, err error) {
 	f.basefmt.Failed(step, match, err)
 	f.stat = failed
-	f.step(f.failed[len(f.failed)-1])
+	f.step(f.failed[len(f.failed) - 1])
 }
 
 func (f *cukefmt) Pending(step *gherkin.Step, match *StepDef) {
 	f.stat = pending
 	f.basefmt.Pending(step, match)
-	f.step(f.pending[len(f.pending)-1])
+	f.step(f.pending[len(f.pending) - 1])
+
+	// the location for pending is the feature file location not the step file.
+	f.curStep.Match.Location = fmt.Sprintf("%s:%d",f.path,step.Location.Line)
+	f.curStep.Result.Duration = nil
 }

suite_test.go

@@ -401,9 +401,27 @@ func (s *suiteContext) theRenderJSONWillBe(docstring *gherkin.DocString) error {
 	expectedArr := expected.([]interface{})
 	actualArr := actual.([]interface{})
 
+	// Created to use in error reporting.
+	expectedCompact := &bytes.Buffer{}
+	actualCompact := &bytes.Buffer{}
+	json.Compact(expectedCompact,[]byte(docstring.Content))
+	json.Compact(actualCompact,s.out.Bytes())
+
 	for idx, entry := range expectedArr {
+
+		// Make sure all of the expected are in the actual
+		if err := s.mapCompareStructure(entry.(map[string]interface{}), actualArr[idx].(map[string]interface{})); err != nil {
+			return fmt.Errorf("err:%v actual result is missing fields: expected:%s actual:%s",err,expectedCompact, actualCompact)
+		}
+
+		// Make sure all of actual are in expected
+		if err := s.mapCompareStructure(actualArr[idx].(map[string]interface{}),entry.(map[string]interface{})); err != nil {
+			return fmt.Errorf("err:%v actual result contains too many fields: expected:%s actual:%s",err,expectedCompact, actualCompact)
+		}
+
+		// Make sure the values are correct
 		if err := s.mapCompare(entry.(map[string]interface{}), actualArr[idx].(map[string]interface{})); err != nil {
-			return err
+			return fmt.Errorf("err:%v values don't match expected:%s actual:%s",err,expectedCompact, actualCompact)
 		}
 	}
 	return nil
@@ -435,10 +453,30 @@ func (s *suiteContext) mapCompare(expected map[string]interface{}, actual map[st
 			}
 		// We need special rules to check location so that we are not bound to version of the code.
 		} else if k == "location" {
+
+			// location is tricky.  the cucumber value is either a the step def location for passed,failed, and skipped.
+			// it is the feature file location for undefined and skipped.
+			// I dont have the result context readily available so the expected input will have
+			// the context i need contained within its value.
+			// FEATURE_PATH myfile.feature:20 or
+			// STEP_ID
+			t := strings.Split(v.(string)," ")
+			if t[0] == "FEATURE_PATH" {
+				if actual[k].(string) != t[1]{
+					return fmt.Errorf("location has unexpected value [%s] should be [%s]",
+						actual[k], t[1])
+				}
+
+			} else if t[0] == "STEP_ID" {
 				if !strings.Contains(actual[k].(string), "suite_test.go:") {
-				return fmt.Errorf("location has unexpected filename [%s] should contains suite_test.go",
+					return fmt.Errorf("location has unexpected filename [%s] should contain suite_test.go",
 						actual[k])
 				}
+
+			} else {
+				return fmt.Errorf("Bad location value [%v]",v)
+			}
+
 		// We need special rules to validate duration too.
 		} else if k == "duration" {
 			if actual[k].(float64) <= 0 {
@@ -465,3 +503,32 @@ func (s *suiteContext) mapCompare(expected map[string]interface{}, actual map[st
 
 	return nil
 }
+/*
+  Due to specialize matching logic to ignore exact matches on the "location" and "duration" fields.  It was
+  necessary to create this compare function to validate the values of the map.
+*/
+func (s *suiteContext) mapCompareStructure(expected map[string]interface{}, actual map[string]interface{}) error {
+
+	// Process all keys in the map and handle them based on the type of the field.
+	for k, v := range expected {
+
+		if actual[k] == nil {
+			return fmt.Errorf("Structure Mismatch: no matching field:[%s] expected value:[%v]",k, v)
+		}
+		// Process other maps via recursion
+		if reflect.TypeOf(v).Kind() == reflect.Map {
+			if err := s.mapCompareStructure(v.(map[string]interface{}), actual[k].(map[string]interface{})); err != nil {
+				return err
+			}
+			// This is an array of maps show as a slice
+		} else if reflect.TypeOf(v).Kind() == reflect.Slice {
+			for i, e := range v.([]interface{}) {
+				if err := s.mapCompareStructure(e.(map[string]interface{}), actual[k].([]interface{})[i].(map[string]interface{})); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}