Discovered defects resulting from not comparing the expected and actual structures. Times when actual contained more fields than expected.

fmt_cucumber.go

@@ -64,7 +64,7 @@ type cukeTag struct {
 type cukeResult struct {
 	Status   string `json:"status"`
 	Error    string `json:"error_message,omitempty"`
-	Duration int    `json:"duration"`
+	Duration *int    `json:"duration,omitempty"`
 }
 
 type cukeMatch struct {
@@ -75,7 +75,7 @@ type cukeStep struct {
 	Keyword   string        `json:"keyword"`
 	Name      string        `json:"name"`
 	Line      int           `json:"line"`
-	Docstring cukeDocstring `json:"doc_string,omitempty"`
+	Docstring *cukeDocstring `json:"doc_string,omitempty"`
 	Match     cukeMatch     `json:"match"`
 	Result    cukeResult    `json:"result"`
 }
@@ -250,17 +250,20 @@ func (f *cukefmt) Summary() {
 
 func (f *cukefmt) step(res *stepResult) {
 
+
 	// determine if test case has finished
 	switch t := f.owner.(type) {
 	case *gherkin.TableRow:
-		f.curStep.Result.Duration = int(time.Since(f.startTime).Nanoseconds())
+		d := int(time.Since(f.startTime).Nanoseconds())
+		f.curStep.Result.Duration = &d
 		f.curStep.Line = t.Location.Line
 		f.curStep.Result.Status = res.typ.String()
 		if res.err != nil {
 			f.curStep.Result.Error = res.err.Error()
 		}
 	case *gherkin.Scenario:
-		f.curStep.Result.Duration = int(time.Since(f.startTime).Nanoseconds())
+		d := int(time.Since(f.startTime).Nanoseconds())
+		f.curStep.Result.Duration = &d
 		f.curStep.Result.Status = res.typ.String()
 		if res.err != nil {
 			f.curStep.Result.Error = res.err.Error()
@@ -279,6 +282,7 @@ func (f *cukefmt) Defined(step *gherkin.Step, def *StepDef) {
 	f.curStep.Keyword = step.Keyword
 
 	if _, ok := step.Argument.(*gherkin.DocString); ok {
+		f.curStep.Docstring = &cukeDocstring{}
 		f.curStep.Docstring.ContentType = strings.TrimSpace(step.Argument.(*gherkin.DocString).ContentType)
 		f.curStep.Docstring.Line = step.Argument.(*gherkin.DocString).Location.Line
 		f.curStep.Docstring.Value = step.Argument.(*gherkin.DocString).Content
@@ -298,12 +302,19 @@ func (f *cukefmt) Passed(step *gherkin.Step, match *StepDef) {
 func (f *cukefmt) Skipped(step *gherkin.Step) {
 	f.basefmt.Skipped(step)
 	f.step(f.skipped[len(f.skipped) - 1])
+
+	// no duration reported for skipped.
+	f.curStep.Result.Duration = nil
 }
 
 func (f *cukefmt) Undefined(step *gherkin.Step) {
 	f.basefmt.Undefined(step)
 	f.stat = undefined
 	f.step(f.undefined[len(f.undefined) - 1])
+
+	// the location for undefined is the feature file location not the step file.
+	f.curStep.Match.Location = fmt.Sprintf("%s:%d",f.path,step.Location.Line)
+	f.curStep.Result.Duration = nil
 }
 
 func (f *cukefmt) Failed(step *gherkin.Step, match *StepDef, err error) {
@@ -316,4 +327,8 @@ func (f *cukefmt) Pending(step *gherkin.Step, match *StepDef) {
 	f.stat = pending
 	f.basefmt.Pending(step, match)
 	f.step(f.pending[len(f.pending) - 1])
+
+	// the location for pending is the feature file location not the step file.
+	f.curStep.Match.Location = fmt.Sprintf("%s:%d",f.path,step.Location.Line)
+	f.curStep.Result.Duration = nil
 }

suite_test.go

@@ -401,9 +401,27 @@ func (s *suiteContext) theRenderJSONWillBe(docstring *gherkin.DocString) error {
 	expectedArr := expected.([]interface{})
 	actualArr := actual.([]interface{})
 
+	// Created to use in error reporting.
+	expectedCompact := &bytes.Buffer{}
+	actualCompact := &bytes.Buffer{}
+	json.Compact(expectedCompact,[]byte(docstring.Content))
+	json.Compact(actualCompact,s.out.Bytes())
+
 	for idx, entry := range expectedArr {
+
+		// Make sure all of the expected are in the actual
+		if err := s.mapCompareStructure(entry.(map[string]interface{}), actualArr[idx].(map[string]interface{})); err != nil {
+			return fmt.Errorf("err:%v actual result is missing fields: expected:%s actual:%s",err,expectedCompact, actualCompact)
+		}
+
+		// Make sure all of actual are in expected
+		if err := s.mapCompareStructure(actualArr[idx].(map[string]interface{}),entry.(map[string]interface{})); err != nil {
+			return fmt.Errorf("err:%v actual result contains too many fields: expected:%s actual:%s",err,expectedCompact, actualCompact)
+		}
+
+		// Make sure the values are correct
 		if err := s.mapCompare(entry.(map[string]interface{}), actualArr[idx].(map[string]interface{})); err != nil {
-			return err
+			return fmt.Errorf("err:%v values don't match expected:%s actual:%s",err,expectedCompact, actualCompact)
 		}
 	}
 	return nil
@@ -435,10 +453,30 @@ func (s *suiteContext) mapCompare(expected map[string]interface{}, actual map[st
 			}
 		// We need special rules to check location so that we are not bound to version of the code.
 		} else if k == "location" {
+
+			// location is tricky.  the cucumber value is either a the step def location for passed,failed, and skipped.
+			// it is the feature file location for undefined and skipped.
+			// I dont have the result context readily available so the expected input will have
+			// the context i need contained within its value.
+			// FEATURE_PATH myfile.feature:20 or
+			// STEP_ID
+			t := strings.Split(v.(string)," ")
+			if t[0] == "FEATURE_PATH" {
+				if actual[k].(string) != t[1]{
+					return fmt.Errorf("location has unexpected value [%s] should be [%s]",
+						actual[k], t[1])
+				}
+
+			} else if t[0] == "STEP_ID" {
 				if !strings.Contains(actual[k].(string), "suite_test.go:") {
-				return fmt.Errorf("location has unexpected filename [%s] should contains suite_test.go",
+					return fmt.Errorf("location has unexpected filename [%s] should contain suite_test.go",
 						actual[k])
 				}
+
+			} else {
+				return fmt.Errorf("Bad location value [%v]",v)
+			}
+
 		// We need special rules to validate duration too.
 		} else if k == "duration" {
 			if actual[k].(float64) <= 0 {
@@ -465,3 +503,32 @@ func (s *suiteContext) mapCompare(expected map[string]interface{}, actual map[st
 
 	return nil
 }
+/*
+  Due to specialize matching logic to ignore exact matches on the "location" and "duration" fields.  It was
+  necessary to create this compare function to validate the values of the map.
+*/
+func (s *suiteContext) mapCompareStructure(expected map[string]interface{}, actual map[string]interface{}) error {
+
+	// Process all keys in the map and handle them based on the type of the field.
+	for k, v := range expected {
+
+		if actual[k] == nil {
+			return fmt.Errorf("Structure Mismatch: no matching field:[%s] expected value:[%v]",k, v)
+		}
+		// Process other maps via recursion
+		if reflect.TypeOf(v).Kind() == reflect.Map {
+			if err := s.mapCompareStructure(v.(map[string]interface{}), actual[k].(map[string]interface{})); err != nil {
+				return err
+			}
+			// This is an array of maps show as a slice
+		} else if reflect.TypeOf(v).Kind() == reflect.Slice {
+			for i, e := range v.([]interface{}) {
+				if err := s.mapCompareStructure(e.(map[string]interface{}), actual[k].([]interface{})[i].(map[string]interface{})); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}