From 9d4b221f7a9d0fd20880fe4e473e28e26f9c357e Mon Sep 17 00:00:00 2001 From: Viacheslav Poturaev Date: Sun, 30 May 2021 22:11:44 +0200 Subject: [PATCH 1/2] Use multiple formatters in the same test run --- internal/formatters/fmt_multi.go | 133 +++++++++++++++++++++++++++++++ run.go | 73 +++++++++-------- 2 files changed, 174 insertions(+), 32 deletions(-) create mode 100644 internal/formatters/fmt_multi.go diff --git a/internal/formatters/fmt_multi.go b/internal/formatters/fmt_multi.go new file mode 100644 index 0000000..a54c80a --- /dev/null +++ b/internal/formatters/fmt_multi.go @@ -0,0 +1,133 @@ +package formatters + +import ( + "io" + + "github.com/cucumber/godog/formatters" + "github.com/cucumber/godog/internal/storage" + "github.com/cucumber/messages-go/v10" +) + +// MultiFormatter passes test progress to multiple formatters. +type MultiFormatter struct { + formatters []formatter + repeater repeater +} + +type formatter struct { + fmt formatters.FormatterFunc + out io.Writer + close bool +} + +type repeater []formatters.Formatter + +type storageFormatter interface { + SetStorage(s *storage.Storage) +} + +// SetStorage passes storage to all added formatters. +func (r repeater) SetStorage(s *storage.Storage) { + for _, f := range r { + if ss, ok := f.(storageFormatter); ok { + ss.SetStorage(s) + } + } +} + +// TestRunStarted triggers TestRunStarted for all added formatters. +func (r repeater) TestRunStarted() { + for _, f := range r { + f.TestRunStarted() + } +} + +// Feature triggers Feature for all added formatters. +func (r repeater) Feature(document *messages.GherkinDocument, s string, bytes []byte) { + for _, f := range r { + f.Feature(document, s, bytes) + } +} + +// Pickle triggers Pickle for all added formatters. +func (r repeater) Pickle(pickle *messages.Pickle) { + for _, f := range r { + f.Pickle(pickle) + } +} + +// Defined triggers Defined for all added formatters. +func (r repeater) Defined(pickle *messages.Pickle, step *messages.Pickle_PickleStep, definition *formatters.StepDefinition) { + for _, f := range r { + f.Defined(pickle, step, definition) + } +} + +// Failed triggers Failed for all added formatters. +func (r repeater) Failed(pickle *messages.Pickle, step *messages.Pickle_PickleStep, definition *formatters.StepDefinition, err error) { + for _, f := range r { + f.Failed(pickle, step, definition, err) + } +} + +// Passed triggers Passed for all added formatters. +func (r repeater) Passed(pickle *messages.Pickle, step *messages.Pickle_PickleStep, definition *formatters.StepDefinition) { + for _, f := range r { + f.Passed(pickle, step, definition) + } +} + +// Skipped triggers Skipped for all added formatters. +func (r repeater) Skipped(pickle *messages.Pickle, step *messages.Pickle_PickleStep, definition *formatters.StepDefinition) { + for _, f := range r { + f.Skipped(pickle, step, definition) + } +} + +// Undefined triggers Undefined for all added formatters. +func (r repeater) Undefined(pickle *messages.Pickle, step *messages.Pickle_PickleStep, definition *formatters.StepDefinition) { + for _, f := range r { + f.Undefined(pickle, step, definition) + } +} + +// Pending triggers Pending for all added formatters. +func (r repeater) Pending(pickle *messages.Pickle, step *messages.Pickle_PickleStep, definition *formatters.StepDefinition) { + for _, f := range r { + f.Pending(pickle, step, definition) + } +} + +// Summary triggers Summary for all added formatters. +func (r repeater) Summary() { + for _, f := range r { + f.Summary() + } +} + +// Add adds formatter with output writer. +func (m *MultiFormatter) Add(name string, out io.Writer) { + f := formatters.FindFmt(name) + if f == nil { + panic("formatter not found: " + name) + } + + m.formatters = append(m.formatters, formatter{ + fmt: f, + out: out, + }) +} + +// FormatterFunc implements the FormatterFunc for the multi formatter. +func (m *MultiFormatter) FormatterFunc(suite string, out io.Writer) formatters.Formatter { + for _, f := range m.formatters { + out := out + if f.out != nil { + out = f.out + } + + m.repeater = append(m.repeater, f.fmt(suite, out)) + } + + return m.repeater +} diff --git a/run.go b/run.go index 0a82200..1a064ab 100644 --- a/run.go +++ b/run.go @@ -16,6 +16,7 @@ import ( "github.com/cucumber/godog/colors" "github.com/cucumber/godog/formatters" + ifmt "github.com/cucumber/godog/internal/formatters" "github.com/cucumber/godog/internal/models" "github.com/cucumber/godog/internal/parser" "github.com/cucumber/godog/internal/storage" @@ -141,28 +142,49 @@ func runWithOptions(suiteName string, runner runner, opt Options) int { output = opt.Output } - if formatterParts := strings.SplitN(opt.Format, ":", 2); len(formatterParts) > 1 { - f, err := os.Create(formatterParts[1]) - if err != nil { - err = fmt.Errorf( - `couldn't create file with name: "%s", error: %s`, - formatterParts[1], err.Error(), - ) - fmt.Fprintln(os.Stderr, err) + multiFmt := ifmt.MultiFormatter{} + for _, formatter := range strings.Split(opt.Format, ",") { + out := output + formatterParts := strings.SplitN(formatter, ":", 2) + + if len(formatterParts) > 1 { + f, err := os.Create(formatterParts[1]) + if err != nil { + err = fmt.Errorf( + `couldn't create file with name: "%s", error: %s`, + formatterParts[1], err.Error(), + ) + fmt.Fprintln(os.Stderr, err) + + return exitOptionError + } + + defer f.Close() + + out = f + } + + if opt.NoColors { + out = colors.Uncolored(out) + } else { + out = colors.Colored(out) + } + + if nil == formatters.FindFmt(formatterParts[0]) { + var names []string + for name := range formatters.AvailableFormatters() { + names = append(names, name) + } + fmt.Fprintln(os.Stderr, fmt.Errorf( + `unregistered formatter name: "%s", use one of: %s`, + opt.Format, + strings.Join(names, ", "), + )) return exitOptionError } - defer f.Close() - - output = f - opt.Format = formatterParts[0] - } - - if opt.NoColors { - output = colors.Uncolored(output) - } else { - output = colors.Colored(output) + multiFmt.Add(formatterParts[0], out) } if opt.ShowStepDefinitions { @@ -184,20 +206,7 @@ func runWithOptions(suiteName string, runner runner, opt Options) int { opt.Concurrency = 1 } - formatter := formatters.FindFmt(opt.Format) - if nil == formatter { - var names []string - for name := range formatters.AvailableFormatters() { - names = append(names, name) - } - fmt.Fprintln(os.Stderr, fmt.Errorf( - `unregistered formatter name: "%s", use one of: %s`, - opt.Format, - strings.Join(names, ", "), - )) - return exitOptionError - } - runner.fmt = formatter(suiteName, output) + runner.fmt = multiFmt.FormatterFunc(suiteName, output) var err error if runner.features, err = parser.ParseFeatures(opt.Tags, opt.Paths); err != nil { From 707025de289e23a72ec20598cc901cacc5a2dc45 Mon Sep 17 00:00:00 2001 From: Viacheslav Poturaev Date: Sun, 30 May 2021 22:32:34 +0200 Subject: [PATCH 2/2] Add test for multi formatter --- internal/formatters/fmt_output_test.go | 2 +- .../formatter-tests/junit,pretty/empty | 5 ++ .../junit,pretty/empty_with_description | 5 ++ .../empty_with_single_scenario_without_steps | 12 +++++ ...gle_scenario_without_steps_and_description | 15 ++++++ .../junit,pretty/scenario_outline | 54 +++++++++++++++++++ .../junit,pretty/scenario_with_background | 18 +++++++ .../scenario_without_steps_with_background | 15 ++++++ .../single_scenario_with_passing_step | 16 ++++++ .../some_scenarions_including_failing | 54 +++++++++++++++++++ .../two_scenarios_with_background_fail | 41 ++++++++++++++ .../junit,pretty/with_few_empty_scenarios | 29 ++++++++++ 12 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 internal/formatters/formatter-tests/junit,pretty/empty create mode 100644 internal/formatters/formatter-tests/junit,pretty/empty_with_description create mode 100644 internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps create mode 100644 internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps_and_description create mode 100644 internal/formatters/formatter-tests/junit,pretty/scenario_outline create mode 100644 internal/formatters/formatter-tests/junit,pretty/scenario_with_background create mode 100644 internal/formatters/formatter-tests/junit,pretty/scenario_without_steps_with_background create mode 100644 internal/formatters/formatter-tests/junit,pretty/single_scenario_with_passing_step create mode 100644 internal/formatters/formatter-tests/junit,pretty/some_scenarions_including_failing create mode 100644 internal/formatters/formatter-tests/junit,pretty/two_scenarios_with_background_fail create mode 100644 internal/formatters/formatter-tests/junit,pretty/with_few_empty_scenarios diff --git a/internal/formatters/fmt_output_test.go b/internal/formatters/fmt_output_test.go index 5a9d689..423558d 100644 --- a/internal/formatters/fmt_output_test.go +++ b/internal/formatters/fmt_output_test.go @@ -25,7 +25,7 @@ func Test_FmtOutput(t *testing.T) { featureFiles, err := listFmtOutputTestsFeatureFiles() require.Nil(t, err) - formatters := []string{"cucumber", "events", "junit", "pretty", "progress"} + formatters := []string{"cucumber", "events", "junit", "pretty", "progress", "junit,pretty"} for _, fmtName := range formatters { for _, featureFile := range featureFiles { diff --git a/internal/formatters/formatter-tests/junit,pretty/empty b/internal/formatters/formatter-tests/junit,pretty/empty new file mode 100644 index 0000000..d2d5f6b --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/empty @@ -0,0 +1,5 @@ + + +No scenarios +No steps +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/empty_with_description b/internal/formatters/formatter-tests/junit,pretty/empty_with_description new file mode 100644 index 0000000..d2d5f6b --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/empty_with_description @@ -0,0 +1,5 @@ + + +No scenarios +No steps +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps b/internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps new file mode 100644 index 0000000..345c16a --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps @@ -0,0 +1,12 @@ +Feature: empty feature + + Scenario: without steps # formatter-tests/features/empty_with_single_scenario_without_steps.feature:3 + + + + + + +1 scenarios (1 undefined) +No steps +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps_and_description b/internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps_and_description new file mode 100644 index 0000000..9abc701 --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/empty_with_single_scenario_without_steps_and_description @@ -0,0 +1,15 @@ +Feature: empty feature + describes + an empty + feature + + Scenario: without steps # formatter-tests/features/empty_with_single_scenario_without_steps_and_description.feature:6 + + + + + + +1 scenarios (1 undefined) +No steps +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/scenario_outline b/internal/formatters/formatter-tests/junit,pretty/scenario_outline new file mode 100644 index 0000000..1f1f92d --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/scenario_outline @@ -0,0 +1,54 @@ +Feature: outline + + Scenario Outline: outline # formatter-tests/features/scenario_outline.feature:5 + Given passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + When passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + Then odd and even number # fmt_output_test.go:103 -> github.com/cucumber/godog/internal/formatters_test.oddEvenStepDef + + Examples: tagged + | odd | even | + | 1 | 2 | + | 2 | 0 | + 2 is not odd + | 3 | 11 | + 11 is not even + + Examples: + | odd | even | + | 1 | 14 | + | 3 | 9 | + 9 is not even + + + + + + + + + + + + + + + + +--- Failed steps: + + Scenario Outline: outline # formatter-tests/features/scenario_outline.feature:5 + Then odd 2 and even 0 number # formatter-tests/features/scenario_outline.feature:8 + Error: 2 is not odd + + Scenario Outline: outline # formatter-tests/features/scenario_outline.feature:5 + Then odd 3 and even 11 number # formatter-tests/features/scenario_outline.feature:8 + Error: 11 is not even + + Scenario Outline: outline # formatter-tests/features/scenario_outline.feature:5 + Then odd 3 and even 9 number # formatter-tests/features/scenario_outline.feature:8 + Error: 9 is not even + + +5 scenarios (2 passed, 3 failed) +15 steps (12 passed, 3 failed) +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/scenario_with_background b/internal/formatters/formatter-tests/junit,pretty/scenario_with_background new file mode 100644 index 0000000..44cb82f --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/scenario_with_background @@ -0,0 +1,18 @@ +Feature: single scenario with background + + Background: named + Given passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + And passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + Scenario: scenario # formatter-tests/features/scenario_with_background.feature:7 + When passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + Then passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + + + + + +1 scenarios (1 passed) +4 steps (4 passed) +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/scenario_without_steps_with_background b/internal/formatters/formatter-tests/junit,pretty/scenario_without_steps_with_background new file mode 100644 index 0000000..410b153 --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/scenario_without_steps_with_background @@ -0,0 +1,15 @@ +Feature: empty feature + + Background: + Given passing step + + Scenario: without steps # formatter-tests/features/scenario_without_steps_with_background.feature:6 + + + + + + +1 scenarios (1 undefined) +No steps +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/single_scenario_with_passing_step b/internal/formatters/formatter-tests/junit,pretty/single_scenario_with_passing_step new file mode 100644 index 0000000..dd33ade --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/single_scenario_with_passing_step @@ -0,0 +1,16 @@ +Feature: single passing scenario + describes + a single scenario + feature + + Scenario: one step passing # formatter-tests/features/single_scenario_with_passing_step.feature:6 + Given a passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + + + + + +1 scenarios (1 passed) +1 steps (1 passed) +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/some_scenarions_including_failing b/internal/formatters/formatter-tests/junit,pretty/some_scenarions_including_failing new file mode 100644 index 0000000..4567095 --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/some_scenarions_including_failing @@ -0,0 +1,54 @@ +Feature: some scenarios + + Scenario: failing # formatter-tests/features/some_scenarions_including_failing.feature:3 + Given passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + When failing step # fmt_output_test.go:117 -> github.com/cucumber/godog/internal/formatters_test.failingStepDef + step failed + Then passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + Scenario: pending # formatter-tests/features/some_scenarions_including_failing.feature:8 + When pending step # fmt_output_test.go:115 -> github.com/cucumber/godog/internal/formatters_test.pendingStepDef + TODO: write pending definition + Then passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + Scenario: undefined # formatter-tests/features/some_scenarions_including_failing.feature:12 + When undefined + Then passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + + + + + + + + + + + + + + + + +--- Failed steps: + + Scenario: failing # formatter-tests/features/some_scenarions_including_failing.feature:3 + When failing step # formatter-tests/features/some_scenarions_including_failing.feature:5 + Error: step failed + + +3 scenarios (1 failed, 1 pending, 1 undefined) +7 steps (1 passed, 1 failed, 1 pending, 1 undefined, 3 skipped) +0s + +You can implement step definitions for undefined steps with these snippets: + +func undefined() error { + return godog.ErrPending +} + +func InitializeScenario(ctx *godog.ScenarioContext) { + ctx.Step(`^undefined$`, undefined) +} + diff --git a/internal/formatters/formatter-tests/junit,pretty/two_scenarios_with_background_fail b/internal/formatters/formatter-tests/junit,pretty/two_scenarios_with_background_fail new file mode 100644 index 0000000..995dec6 --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/two_scenarios_with_background_fail @@ -0,0 +1,41 @@ +Feature: two scenarios with background fail + + Background: + Given passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + And failing step # fmt_output_test.go:117 -> github.com/cucumber/godog/internal/formatters_test.failingStepDef + step failed + + Scenario: one # formatter-tests/features/two_scenarios_with_background_fail.feature:7 + When passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + Then passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + Scenario: two # formatter-tests/features/two_scenarios_with_background_fail.feature:11 + Then passing step # fmt_output_test.go:101 -> github.com/cucumber/godog/internal/formatters_test.passingStepDef + + + + + + + + + + + + + + +--- Failed steps: + + Scenario: one # formatter-tests/features/two_scenarios_with_background_fail.feature:7 + And failing step # formatter-tests/features/two_scenarios_with_background_fail.feature:5 + Error: step failed + + Scenario: two # formatter-tests/features/two_scenarios_with_background_fail.feature:11 + And failing step # formatter-tests/features/two_scenarios_with_background_fail.feature:5 + Error: step failed + + +2 scenarios (2 failed) +7 steps (2 passed, 2 failed, 3 skipped) +0s diff --git a/internal/formatters/formatter-tests/junit,pretty/with_few_empty_scenarios b/internal/formatters/formatter-tests/junit,pretty/with_few_empty_scenarios new file mode 100644 index 0000000..752f37e --- /dev/null +++ b/internal/formatters/formatter-tests/junit,pretty/with_few_empty_scenarios @@ -0,0 +1,29 @@ +Feature: few empty scenarios + + Scenario: one # formatter-tests/features/with_few_empty_scenarios.feature:3 + + Scenario Outline: two # formatter-tests/features/with_few_empty_scenarios.feature:5 + + Examples: first group + | one | two | + | 1 | 2 | + | 4 | 7 | + + Examples: second group + | one | two | + | 5 | 9 | + + Scenario: three # formatter-tests/features/with_few_empty_scenarios.feature:16 + + + + + + + + + + +5 scenarios (5 undefined) +No steps +0s