Merge pull request #272 from cucumber/improved-verification-of-concurrent-runs
Added comparison between single and multi threaded runs
Этот коммит содержится в:
Fredrik Lönnblad
GitHub
коммит
c9206b43f6
4 изменённых файлов: 110 добавлений и 151 удалений
|
|
@ -1,108 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuites name="succeed" tests="79" skipped="0" failures="0" errors="0" time="0">
|
||||
<testsuite name="JUnit XML formatter" tests="9" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="Support of Feature Plus Scenario Node" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Node With Tags" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline With Tags" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario With Steps" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline With Steps" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Comments" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Docstrings" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Undefined, Pending and Skipped status" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="cucumber json formatter" tests="9" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="Support of Feature Plus Scenario Node" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Node With Tags" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline With Tags" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario With Steps" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline With Steps" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Comments" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Docstrings" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Undefined, Pending and Skipped status" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="event stream formatter" tests="3" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="should fire only suite events without any scenario" status="passed" time="0"></testcase>
|
||||
<testcase name="should process simple scenario" status="passed" time="0"></testcase>
|
||||
<testcase name="should process outline scenario" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="load features" tests="6" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="load features within path" status="passed" time="0"></testcase>
|
||||
<testcase name="load a specific feature file" status="passed" time="0"></testcase>
|
||||
<testcase name="loaded feature should have a number of scenarios #1" status="passed" time="0"></testcase>
|
||||
<testcase name="loaded feature should have a number of scenarios #2" status="passed" time="0"></testcase>
|
||||
<testcase name="loaded feature should have a number of scenarios #3" status="passed" time="0"></testcase>
|
||||
<testcase name="load a number of feature files" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="pretty formatter" tests="10" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="Support of Feature Plus Scenario Node" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Node With Tags" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline With Tags" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario With Steps" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Feature Plus Scenario Outline With Steps" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Comments" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Docstrings" status="passed" time="0"></testcase>
|
||||
<testcase name="Support of Undefined, Pending and Skipped status" status="passed" time="0"></testcase>
|
||||
<testcase name="Support data injection in BeforeStep" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="run background" tests="3" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="should run background steps" status="passed" time="0"></testcase>
|
||||
<testcase name="should skip all consequent steps on failure" status="passed" time="0"></testcase>
|
||||
<testcase name="should continue undefined steps" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="run features" tests="11" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="should run a normal feature" status="passed" time="0"></testcase>
|
||||
<testcase name="should skip steps after failure" status="passed" time="0"></testcase>
|
||||
<testcase name="should skip all scenarios if background fails" status="passed" time="0"></testcase>
|
||||
<testcase name="should skip steps after undefined" status="passed" time="0"></testcase>
|
||||
<testcase name="should match undefined steps in a row" status="passed" time="0"></testcase>
|
||||
<testcase name="should skip steps on pending" status="passed" time="0"></testcase>
|
||||
<testcase name="should handle pending step" status="passed" time="0"></testcase>
|
||||
<testcase name="should mark undefined steps after pending" status="passed" time="0"></testcase>
|
||||
<testcase name="should fail suite if undefined steps follow after the failure" status="passed" time="0"></testcase>
|
||||
<testcase name="should fail suite and skip pending step after failed step" status="passed" time="0"></testcase>
|
||||
<testcase name="should fail suite and skip next step after failed step" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="run features with nested steps" tests="6" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="should run passing multistep successfully" status="passed" time="0"></testcase>
|
||||
<testcase name="should fail multistep" status="passed" time="0"></testcase>
|
||||
<testcase name="should fail nested multistep" status="passed" time="0"></testcase>
|
||||
<testcase name="should skip steps after undefined multistep" status="passed" time="0"></testcase>
|
||||
<testcase name="should match undefined steps in a row" status="passed" time="0"></testcase>
|
||||
<testcase name="should mark undefined steps after pending" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="run outline" tests="6" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="should run a normal outline" status="passed" time="0"></testcase>
|
||||
<testcase name="should continue through examples on failure" status="passed" time="0"></testcase>
|
||||
<testcase name="should skip examples on background failure" status="passed" time="0"></testcase>
|
||||
<testcase name="should translate step table body" status="passed" time="0"></testcase>
|
||||
<testcase name="should translate step doc string argument #1" status="passed" time="0"></testcase>
|
||||
<testcase name="should translate step doc string argument #2" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="suite events" tests="6" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="triggers before scenario event" status="passed" time="0"></testcase>
|
||||
<testcase name="triggers appropriate events for a single scenario" status="passed" time="0"></testcase>
|
||||
<testcase name="triggers appropriate events whole feature" status="passed" time="0"></testcase>
|
||||
<testcase name="triggers appropriate events for two feature files" status="passed" time="0"></testcase>
|
||||
<testcase name="should not trigger events on empty feature" status="passed" time="0"></testcase>
|
||||
<testcase name="should not trigger events on empty scenarios" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="tag filters" tests="4" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="should filter outline examples by tags" status="passed" time="0"></testcase>
|
||||
<testcase name="should filter scenarios by X tag" status="passed" time="0"></testcase>
|
||||
<testcase name="should filter scenarios by X tag not having Y" status="passed" time="0"></testcase>
|
||||
<testcase name="should filter scenarios having Y and Z tags" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="undefined step snippets" tests="5" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="should generate snippets" status="passed" time="0"></testcase>
|
||||
<testcase name="should generate snippets with more arguments" status="passed" time="0"></testcase>
|
||||
<testcase name="should handle escaped symbols" status="passed" time="0"></testcase>
|
||||
<testcase name="should handle string argument followed by comma" status="passed" time="0"></testcase>
|
||||
<testcase name="should handle arguments in the beggining or end of the step" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
<testsuite name="užkrauti savybes" tests="1" skipped="0" failures="0" errors="0" time="0">
|
||||
<testcase name="savybių užkrovimas iš aplanko" status="passed" time="0"></testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
...................................................................... 70
|
||||
...................................................................... 140
|
||||
...................................................................... 210
|
||||
...................................................................... 280
|
||||
.......................... 306
|
||||
|
||||
|
||||
79 scenarios (79 passed)
|
||||
306 steps (306 passed)
|
||||
0s
|
||||
8
run.go
8
run.go
|
|
@ -75,7 +75,9 @@ func (r *runner) concurrent(rate int, formatterFn func() Formatter) (failed bool
|
|||
r.initializer(suite)
|
||||
suite.run()
|
||||
if suite.failed {
|
||||
copyLock.Lock()
|
||||
*fail = true
|
||||
copyLock.Unlock()
|
||||
}
|
||||
if useFmtCopy {
|
||||
copyLock.Lock()
|
||||
|
|
@ -274,7 +276,11 @@ func supportsConcurrency(format string) bool {
|
|||
switch format {
|
||||
case "progress", "junit":
|
||||
return true
|
||||
case "events", "pretty", "cucumber":
|
||||
case "events":
|
||||
return false
|
||||
case "cucumber":
|
||||
return false
|
||||
case "pretty":
|
||||
return false
|
||||
default:
|
||||
return true // enables concurrent custom formatters to work
|
||||
|
|
|
|||
135
run_test.go
135
run_test.go
|
|
@ -6,7 +6,6 @@ import (
|
|||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
|
|
@ -250,56 +249,128 @@ func TestFeatureFilePathParser(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
type succeedRunTestCase struct {
|
||||
format string // formatter to use
|
||||
concurrency int // concurrency option range to test
|
||||
filename string // expected output file
|
||||
func TestAllFeaturesRun(t *testing.T) {
|
||||
const concurrency = 10
|
||||
const format = "progress"
|
||||
|
||||
const expected = `...................................................................... 70
|
||||
...................................................................... 140
|
||||
...................................................................... 210
|
||||
...................................................................... 280
|
||||
.......................... 306
|
||||
|
||||
|
||||
79 scenarios (79 passed)
|
||||
306 steps (306 passed)
|
||||
0s
|
||||
`
|
||||
|
||||
actualStatus, actualOutput := testRunWithOptions(t, format, concurrency, []string{"features"})
|
||||
|
||||
assert.Equal(t, exitSuccess, actualStatus)
|
||||
assert.Equal(t, expected, actualOutput)
|
||||
}
|
||||
|
||||
func TestConcurrencyRun(t *testing.T) {
|
||||
testCases := []succeedRunTestCase{
|
||||
{format: "progress", concurrency: 4, filename: "fixtures/progress_output.txt"},
|
||||
{format: "junit", concurrency: 4, filename: "fixtures/junit_output.xml"},
|
||||
func TestFormatterConcurrencyRun(t *testing.T) {
|
||||
formatters := []string{
|
||||
"progress",
|
||||
"junit",
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
expectedOutput, err := ioutil.ReadFile(tc.filename)
|
||||
require.NoError(t, err)
|
||||
featurePaths := []string{"formatter-tests/features"}
|
||||
|
||||
for concurrency := range make([]int, tc.concurrency) {
|
||||
t.Run(
|
||||
fmt.Sprintf("%s/concurrency/%d", tc.format, concurrency),
|
||||
func(t *testing.T) {
|
||||
testSucceedRun(t, tc.format, concurrency, string(expectedOutput))
|
||||
},
|
||||
)
|
||||
}
|
||||
const concurrency = 10
|
||||
|
||||
for _, formatter := range formatters {
|
||||
t.Run(
|
||||
fmt.Sprintf("%s/concurrency/%d", formatter, concurrency),
|
||||
func(t *testing.T) {
|
||||
singleThreadStatus, singleThreadOutput := testRunWithOptions(t, formatter, 1, featurePaths)
|
||||
status, actual := testRunWithOptions(t, formatter, concurrency, featurePaths)
|
||||
|
||||
assert.Equal(t, singleThreadStatus, status)
|
||||
assertConcurrencyOutput(t, formatter, singleThreadOutput, actual)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func testSucceedRun(t *testing.T, format string, concurrency int, expected string) {
|
||||
func testRunWithOptions(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) {
|
||||
output := new(bytes.Buffer)
|
||||
|
||||
opt := Options{
|
||||
Format: format,
|
||||
NoColors: true,
|
||||
Paths: []string{"features"},
|
||||
Paths: featurePaths,
|
||||
Concurrency: concurrency,
|
||||
Output: output,
|
||||
}
|
||||
|
||||
status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opt)
|
||||
assert.Equal(t, exitSuccess, status)
|
||||
|
||||
b, err := ioutil.ReadAll(output)
|
||||
actual, err := ioutil.ReadAll(output)
|
||||
require.NoError(t, err)
|
||||
|
||||
suiteCtxReg := regexp.MustCompile(`suite_context.go:\d+`)
|
||||
|
||||
expected = suiteCtxReg.ReplaceAllString(expected, `suite_context.go:0`)
|
||||
|
||||
actual := strings.TrimSpace(string(b))
|
||||
actual = suiteCtxReg.ReplaceAllString(actual, `suite_context.go:0`)
|
||||
|
||||
assert.Equalf(t, expected, actual, "[%s]", actual)
|
||||
return status, string(actual)
|
||||
}
|
||||
|
||||
func assertConcurrencyOutput(t *testing.T, formatter string, expected, actual string) {
|
||||
switch formatter {
|
||||
case "cucumber", "junit", "pretty", "events":
|
||||
expectedRows := strings.Split(expected, "\n")
|
||||
actualRows := strings.Split(actual, "\n")
|
||||
assert.ElementsMatch(t, expectedRows, actualRows)
|
||||
case "progress":
|
||||
expectedOutput := parseProgressOutput(expected)
|
||||
actualOutput := parseProgressOutput(actual)
|
||||
|
||||
assert.Equal(t, expectedOutput.passed, actualOutput.passed)
|
||||
assert.Equal(t, expectedOutput.skipped, actualOutput.skipped)
|
||||
assert.Equal(t, expectedOutput.failed, actualOutput.failed)
|
||||
assert.Equal(t, expectedOutput.undefined, actualOutput.undefined)
|
||||
assert.Equal(t, expectedOutput.pending, actualOutput.pending)
|
||||
assert.Equal(t, expectedOutput.noOfStepsPerRow, actualOutput.noOfStepsPerRow)
|
||||
assert.ElementsMatch(t, expectedOutput.bottomRows, actualOutput.bottomRows)
|
||||
}
|
||||
}
|
||||
|
||||
func parseProgressOutput(output string) (parsed progressOutput) {
|
||||
mainParts := strings.Split(output, "\n\n\n")
|
||||
|
||||
topRows := strings.Split(mainParts[0], "\n")
|
||||
parsed.bottomRows = strings.Split(mainParts[1], "\n")
|
||||
|
||||
parsed.noOfStepsPerRow = make([]string, len(topRows))
|
||||
for idx, row := range topRows {
|
||||
rowParts := strings.Split(row, " ")
|
||||
stepResults := strings.Split(rowParts[0], "")
|
||||
parsed.noOfStepsPerRow[idx] = rowParts[1]
|
||||
|
||||
for _, stepResult := range stepResults {
|
||||
switch stepResult {
|
||||
case ".":
|
||||
parsed.passed++
|
||||
case "-":
|
||||
parsed.skipped++
|
||||
case "F":
|
||||
parsed.failed++
|
||||
case "U":
|
||||
parsed.undefined++
|
||||
case "P":
|
||||
parsed.pending++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parsed
|
||||
}
|
||||
|
||||
type progressOutput struct {
|
||||
passed int
|
||||
skipped int
|
||||
failed int
|
||||
undefined int
|
||||
pending int
|
||||
noOfStepsPerRow []string
|
||||
bottomRows []string
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Создание таблицы
Сослаться в новой задаче