Merge pull request #214 from lonnblad/testing/added-cucumber-run-tests

Added run tests for Cucumber

fixtures/junit_output.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="succeed" tests="60" skipped="0" failures="0" errors="0" time="0s">
+  <testsuite name="cucumber json formatter" tests="9" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="Support of Feature Plus Scenario Node" status="passed" time="0s"></testcase>
+    <testcase name="Support of Feature Plus Scenario Node With Tags" status="passed" time="0s"></testcase>
+    <testcase name="Support of Feature Plus Scenario Outline" status="passed" time="0s"></testcase>
+    <testcase name="Support of Feature Plus Scenario Outline With Tags" status="passed" time="0s"></testcase>
+    <testcase name="Support of Feature Plus Scenario With Steps" status="passed" time="0s"></testcase>
+    <testcase name="Support of Feature Plus Scenario Outline With Steps" status="passed" time="0s"></testcase>
+    <testcase name="Support of Comments" status="passed" time="0s"></testcase>
+    <testcase name="Support of Docstrings" status="passed" time="0s"></testcase>
+    <testcase name="Support of Undefined, Pending and Skipped status" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="event stream formatter" tests="3" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="should fire only suite events without any scenario" status="passed" time="0s"></testcase>
+    <testcase name="should process simple scenario" status="passed" time="0s"></testcase>
+    <testcase name="should process outline scenario" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="load features" tests="6" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="load features within path" status="passed" time="0s"></testcase>
+    <testcase name="load a specific feature file" status="passed" time="0s"></testcase>
+    <testcase name="loaded feature should have a number of scenarios #1" status="passed" time="0s"></testcase>
+    <testcase name="loaded feature should have a number of scenarios #2" status="passed" time="0s"></testcase>
+    <testcase name="loaded feature should have a number of scenarios #3" status="passed" time="0s"></testcase>
+    <testcase name="load a number of feature files" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="run background" tests="3" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="should run background steps" status="passed" time="0s"></testcase>
+    <testcase name="should skip all consequent steps on failure" status="passed" time="0s"></testcase>
+    <testcase name="should continue undefined steps" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="run features" tests="11" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="should run a normal feature" status="passed" time="0s"></testcase>
+    <testcase name="should skip steps after failure" status="passed" time="0s"></testcase>
+    <testcase name="should skip all scenarios if background fails" status="passed" time="0s"></testcase>
+    <testcase name="should skip steps after undefined" status="passed" time="0s"></testcase>
+    <testcase name="should match undefined steps in a row" status="passed" time="0s"></testcase>
+    <testcase name="should skip steps on pending" status="passed" time="0s"></testcase>
+    <testcase name="should handle pending step" status="passed" time="0s"></testcase>
+    <testcase name="should mark undefined steps after pending" status="passed" time="0s"></testcase>
+    <testcase name="should fail suite if undefined steps follow after the failure" status="passed" time="0s"></testcase>
+    <testcase name="should fail suite and skip pending step after failed step" status="passed" time="0s"></testcase>
+    <testcase name="should fail suite and skip next step after failed step" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="run features with nested steps" tests="6" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="should run passing multistep successfully" status="passed" time="0s"></testcase>
+    <testcase name="should fail multistep" status="passed" time="0s"></testcase>
+    <testcase name="should fail nested multistep" status="passed" time="0s"></testcase>
+    <testcase name="should skip steps after undefined multistep" status="passed" time="0s"></testcase>
+    <testcase name="should match undefined steps in a row" status="passed" time="0s"></testcase>
+    <testcase name="should mark undefined steps after pending" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="run outline" tests="6" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="should run a normal outline" status="passed" time="0s"></testcase>
+    <testcase name="should continue through examples on failure" status="passed" time="0s"></testcase>
+    <testcase name="should skip examples on background failure" status="passed" time="0s"></testcase>
+    <testcase name="should translate step table body" status="passed" time="0s"></testcase>
+    <testcase name="should translate step doc string argument #1" status="passed" time="0s"></testcase>
+    <testcase name="should translate step doc string argument #2" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="suite events" tests="6" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="triggers before scenario event" status="passed" time="0s"></testcase>
+    <testcase name="triggers appropriate events for a single scenario" status="passed" time="0s"></testcase>
+    <testcase name="triggers appropriate events whole feature" status="passed" time="0s"></testcase>
+    <testcase name="triggers appropriate events for two feature files" status="passed" time="0s"></testcase>
+    <testcase name="should not trigger events on empty feature" status="passed" time="0s"></testcase>
+    <testcase name="should not trigger events on empty scenarios" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="tag filters" tests="4" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="should filter outline examples by tags" status="passed" time="0s"></testcase>
+    <testcase name="should filter scenarios by X tag" status="passed" time="0s"></testcase>
+    <testcase name="should filter scenarios by X tag not having Y" status="passed" time="0s"></testcase>
+    <testcase name="should filter scenarios having Y and Z tags" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="undefined step snippets" tests="5" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="should generate snippets" status="passed" time="0s"></testcase>
+    <testcase name="should generate snippets with more arguments" status="passed" time="0s"></testcase>
+    <testcase name="should handle escaped symbols" status="passed" time="0s"></testcase>
+    <testcase name="should handle string argument followed by comma" status="passed" time="0s"></testcase>
+    <testcase name="should handle arguments in the beggining or end of the step" status="passed" time="0s"></testcase>
+  </testsuite>
+  <testsuite name="užkrauti savybes" tests="1" skipped="0" failures="0" errors="0" time="0s">
+    <testcase name="savybių užkrovimas iš aplanko" status="passed" time="0s"></testcase>
+  </testsuite>
+</testsuites>

fixtures/progress_output.txt

@@ -0,0 +1,9 @@
+...................................................................... 70
+...................................................................... 140
+...................................................................... 210
+.......................................                                249
+
+
+60 scenarios (60 passed)
+249 steps (249 passed)
+0s

go.sum

@@ -2,4 +2,5 @@ github.com/DATA-DOG/go-txdb v0.1.3 h1:R4v6OuOcy2O147e2zHxU0B4NDtF+INb5R9q/CV7AEM
 github.com/DATA-DOG/go-txdb v0.1.3/go.mod h1:DhAhxMXZpUJVGnT+p9IbzJoRKvlArO2pkHjnGX7o0n0=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=

run_test.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/cucumber/godog/colors"
 	"github.com/cucumber/godog/gherkin"
@@ -277,142 +276,47 @@ func TestFeatureFilePathParser(t *testing.T) {
 	}
 }
 
-func TestSucceedWithProgressAndConcurrencyOption(t *testing.T) {
+type succeedRunTestCase struct {
+	format      string // formatter to use
+	concurrency int    // concurrency option range to test
+	filename    string // expected output file
+}
+
+func TestSucceedRun(t *testing.T) {
+	testCases := []succeedRunTestCase{
+		{format: "progress", concurrency: 4, filename: "fixtures/progress_output.txt"},
+		{format: "junit", concurrency: 4, filename: "fixtures/junit_output.xml"},
+		{format: "cucumber", concurrency: 2, filename: "fixtures/cucumber_output.json"},
+	}
+
+	for _, tc := range testCases {
+		expectedOutput, err := ioutil.ReadFile(tc.filename)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for concurrency := range make([]int, tc.concurrency) {
+			t.Run(
+				fmt.Sprintf("%s/concurrency/%d", tc.format, concurrency),
+				func(t *testing.T) {
+					testSucceedRun(t, tc.format, concurrency, string(expectedOutput))
+				},
+			)
+		}
+	}
+}
+
+func testSucceedRun(t *testing.T, format string, concurrency int, expectedOutput string) {
 	output := new(bytes.Buffer)
 
 	opt := Options{
-		Format:      "progress",
+		Format:      format,
 		NoColors:    true,
 		Paths:       []string{"features"},
-		Concurrency: 2,
+		Concurrency: concurrency,
 		Output:      output,
 	}
 
-	expectedOutput := `...................................................................... 70
-...................................................................... 140
-...................................................................... 210
-.......................................                                249
-
-
-60 scenarios (60 passed)
-249 steps (249 passed)
-0s`
-
-	status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opt)
-	if status != exitSuccess {
-		t.Fatalf("expected exit status to be 0, but was: %d", status)
-	}
-
-	b, err := ioutil.ReadAll(output)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	out := strings.TrimSpace(string(b))
-	if out != expectedOutput {
-		t.Fatalf("unexpected output: \"%s\"", out)
-	}
-}
-
-func TestSucceedWithJunitAndConcurrencyOption(t *testing.T) {
-	output := new(bytes.Buffer)
-
-	opt := Options{
-		Format:      "junit",
-		NoColors:    true,
-		Paths:       []string{"features"},
-		Concurrency: 2,
-		Output:      output,
-	}
-
-	zeroSecondsString := (0 * time.Second).String()
-
-	expectedOutput := `<?xml version="1.0" encoding="UTF-8"?>
-<testsuites name="succeed" tests="60" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-  <testsuite name="cucumber json formatter" tests="9" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="Support of Feature Plus Scenario Node" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Feature Plus Scenario Node With Tags" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Feature Plus Scenario Outline" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Feature Plus Scenario Outline With Tags" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Feature Plus Scenario With Steps" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Feature Plus Scenario Outline With Steps" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Comments" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Docstrings" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="Support of Undefined, Pending and Skipped status" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="event stream formatter" tests="3" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="should fire only suite events without any scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should process simple scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should process outline scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="load features" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="load features within path" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="load a specific feature file" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="loaded feature should have a number of scenarios #1" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="loaded feature should have a number of scenarios #2" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="loaded feature should have a number of scenarios #3" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="load a number of feature files" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="run background" tests="3" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="should run background steps" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should skip all consequent steps on failure" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should continue undefined steps" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="run features" tests="11" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="should run a normal feature" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should skip steps after failure" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should skip all scenarios if background fails" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should skip steps after undefined" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should match undefined steps in a row" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should skip steps on pending" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should handle pending step" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should mark undefined steps after pending" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should fail suite if undefined steps follow after the failure" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should fail suite and skip pending step after failed step" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should fail suite and skip next step after failed step" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="run features with nested steps" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="should run passing multistep successfully" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should fail multistep" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should fail nested multistep" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should skip steps after undefined multistep" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should match undefined steps in a row" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should mark undefined steps after pending" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="run outline" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="should run a normal outline" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should continue through examples on failure" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should skip examples on background failure" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should translate step table body" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should translate step doc string argument #1" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should translate step doc string argument #2" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="suite events" tests="6" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="triggers before scenario event" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="triggers appropriate events for a single scenario" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="triggers appropriate events whole feature" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="triggers appropriate events for two feature files" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should not trigger events on empty feature" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should not trigger events on empty scenarios" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="tag filters" tests="4" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="should filter outline examples by tags" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should filter scenarios by X tag" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should filter scenarios by X tag not having Y" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should filter scenarios having Y and Z tags" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="undefined step snippets" tests="5" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="should generate snippets" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should generate snippets with more arguments" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should handle escaped symbols" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should handle string argument followed by comma" status="passed" time="` + zeroSecondsString + `"></testcase>
-    <testcase name="should handle arguments in the beggining or end of the step" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-  <testsuite name="užkrauti savybes" tests="1" skipped="0" failures="0" errors="0" time="` + zeroSecondsString + `">
-    <testcase name="savybių užkrovimas iš aplanko" status="passed" time="` + zeroSecondsString + `"></testcase>
-  </testsuite>
-</testsuites>`
-
 	status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opt)
 	if status != exitSuccess {
 		t.Fatalf("expected exit status to be 0, but was: %d", status)