From 11f28803ec5d05322d29edae4d71a046e93d5b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20L=C3=B6nnblad?= Date: Tue, 24 Mar 2020 11:37:53 +0100 Subject: [PATCH] Added comparison between single and multi threaded runs --- fixtures/junit_output.xml | 108 ---------------------------- fixtures/progress_output.txt | 10 --- run.go | 8 ++- run_test.go | 135 ++++++++++++++++++++++++++--------- 4 files changed, 110 insertions(+), 151 deletions(-) delete mode 100644 fixtures/junit_output.xml delete mode 100644 fixtures/progress_output.txt diff --git a/fixtures/junit_output.xml b/fixtures/junit_output.xml deleted file mode 100644 index 0bdc75b..0000000 --- a/fixtures/junit_output.xml +++ /dev/null @@ -1,108 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/fixtures/progress_output.txt b/fixtures/progress_output.txt deleted file mode 100644 index 1c45625..0000000 --- a/fixtures/progress_output.txt +++ /dev/null @@ -1,10 +0,0 @@ -...................................................................... 70 -...................................................................... 140 -...................................................................... 210 -...................................................................... 280 -.......................... 306 - - -79 scenarios (79 passed) -306 steps (306 passed) -0s \ No newline at end of file diff --git a/run.go b/run.go index 8af23f4..7cfc4f9 100644 --- a/run.go +++ b/run.go @@ -75,7 +75,9 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool r.initializer(suite) suite.run() if suite.failed { + copyLock.Lock() *fail = true + copyLock.Unlock() } if useFmtCopy { copyLock.Lock() @@ -274,7 +276,11 @@ func supportsConcurrency(format string) bool { switch format { case "progress", "junit": return true - case "events", "pretty", "cucumber": + case "events": + return false + case "cucumber": + return false + case "pretty": return false default: return true // enables concurrent custom formatters to work diff --git a/run_test.go b/run_test.go index 3db7648..85bd224 100644 --- a/run_test.go +++ b/run_test.go @@ -6,7 +6,6 @@ import ( "io" "io/ioutil" "os" - "regexp" "strings" "testing" @@ -250,56 +249,128 @@ func TestFeatureFilePathParser(t *testing.T) { } } -type succeedRunTestCase struct { - format string // formatter to use - concurrency int // concurrency option range to test - filename string // expected output file +func TestAllFeaturesRun(t *testing.T) { + const concurrency = 10 + const format = "progress" + + const expected = `...................................................................... 70 +...................................................................... 140 +...................................................................... 210 +...................................................................... 280 +.......................... 306 + + +79 scenarios (79 passed) +306 steps (306 passed) +0s +` + + actualStatus, actualOutput := testRunWithOptions(t, format, concurrency, []string{"features"}) + + assert.Equal(t, exitSuccess, actualStatus) + assert.Equal(t, expected, actualOutput) } -func TestConcurrencyRun(t *testing.T) { - testCases := []succeedRunTestCase{ - {format: "progress", concurrency: 4, filename: "fixtures/progress_output.txt"}, - {format: "junit", concurrency: 4, filename: "fixtures/junit_output.xml"}, +func TestFormatterConcurrencyRun(t *testing.T) { + formatters := []string{ + "progress", + "junit", } - for _, tc := range testCases { - expectedOutput, err := ioutil.ReadFile(tc.filename) - require.NoError(t, err) + featurePaths := []string{"formatter-tests/features"} - for concurrency := range make([]int, tc.concurrency) { - t.Run( - fmt.Sprintf("%s/concurrency/%d", tc.format, concurrency), - func(t *testing.T) { - testSucceedRun(t, tc.format, concurrency, string(expectedOutput)) - }, - ) - } + const concurrency = 10 + + for _, formatter := range formatters { + t.Run( + fmt.Sprintf("%s/concurrency/%d", formatter, concurrency), + func(t *testing.T) { + singleThreadStatus, singleThreadOutput := testRunWithOptions(t, formatter, 1, featurePaths) + status, actual := testRunWithOptions(t, formatter, concurrency, featurePaths) + + assert.Equal(t, singleThreadStatus, status) + assertConcurrencyOutput(t, formatter, singleThreadOutput, actual) + }, + ) } } -func testSucceedRun(t *testing.T, format string, concurrency int, expected string) { +func testRunWithOptions(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) { output := new(bytes.Buffer) opt := Options{ Format: format, NoColors: true, - Paths: []string{"features"}, + Paths: featurePaths, Concurrency: concurrency, Output: output, } status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opt) - assert.Equal(t, exitSuccess, status) - b, err := ioutil.ReadAll(output) + actual, err := ioutil.ReadAll(output) require.NoError(t, err) - suiteCtxReg := regexp.MustCompile(`suite_context.go:\d+`) - - expected = suiteCtxReg.ReplaceAllString(expected, `suite_context.go:0`) - - actual := strings.TrimSpace(string(b)) - actual = suiteCtxReg.ReplaceAllString(actual, `suite_context.go:0`) - - assert.Equalf(t, expected, actual, "[%s]", actual) + return status, string(actual) +} + +func assertConcurrencyOutput(t *testing.T, formatter string, expected, actual string) { + switch formatter { + case "cucumber", "junit", "pretty", "events": + expectedRows := strings.Split(expected, "\n") + actualRows := strings.Split(actual, "\n") + assert.ElementsMatch(t, expectedRows, actualRows) + case "progress": + expectedOutput := parseProgressOutput(expected) + actualOutput := parseProgressOutput(actual) + + assert.Equal(t, expectedOutput.passed, actualOutput.passed) + assert.Equal(t, expectedOutput.skipped, actualOutput.skipped) + assert.Equal(t, expectedOutput.failed, actualOutput.failed) + assert.Equal(t, expectedOutput.undefined, actualOutput.undefined) + assert.Equal(t, expectedOutput.pending, actualOutput.pending) + assert.Equal(t, expectedOutput.noOfStepsPerRow, actualOutput.noOfStepsPerRow) + assert.ElementsMatch(t, expectedOutput.bottomRows, actualOutput.bottomRows) + } +} + +func parseProgressOutput(output string) (parsed progressOutput) { + mainParts := strings.Split(output, "\n\n\n") + + topRows := strings.Split(mainParts[0], "\n") + parsed.bottomRows = strings.Split(mainParts[1], "\n") + + parsed.noOfStepsPerRow = make([]string, len(topRows)) + for idx, row := range topRows { + rowParts := strings.Split(row, " ") + stepResults := strings.Split(rowParts[0], "") + parsed.noOfStepsPerRow[idx] = rowParts[1] + + for _, stepResult := range stepResults { + switch stepResult { + case ".": + parsed.passed++ + case "-": + parsed.skipped++ + case "F": + parsed.failed++ + case "U": + parsed.undefined++ + case "P": + parsed.pending++ + } + } + } + + return parsed +} + +type progressOutput struct { + passed int + skipped int + failed int + undefined int + pending int + noOfStepsPerRow []string + bottomRows []string }