diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index 8a409382..ba1bfd30 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -7,6 +7,7 @@ package testing import ( + "flag" "fmt" "io" "math" @@ -14,8 +15,13 @@ import ( "time" ) +func initBenchmarkFlags() { + matchBenchmarks = flag.String("test.bench", "", "run only benchmarks matching `regexp`") +} + var ( - benchTime = benchTimeFlag{d: 1 * time.Second} // changed during test of testing package + matchBenchmarks *string + benchTime = benchTimeFlag{d: 1 * time.Second} // changed during test of testing package ) type benchTimeFlag struct { @@ -271,17 +277,27 @@ func prettyPrint(w io.Writer, x float64, unit string) { } type benchContext struct { + match *matcher + maxLen int // The largest recorded benchmark name. } -func runBenchmarks(benchmarks []InternalBenchmark) bool { - if len(benchmarks) == 0 { +func runBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) bool { + // If no flag was specified, don't run benchmarks. + if len(*matchBenchmarks) == 0 { return true } - ctx := &benchContext{} + ctx := &benchContext{ + match: newMatcher(matchString, *matchBenchmarks, "-test.bench"), + } + var bs []InternalBenchmark for _, Benchmark := range benchmarks { - if l := len(Benchmark.Name); l > ctx.maxLen { - ctx.maxLen = l + if _, matched, _ := ctx.match.fullName(nil, Benchmark.Name); matched { + bs = append(bs, Benchmark) + benchName := Benchmark.Name + if l := len(benchName); l > ctx.maxLen { + ctx.maxLen = l + } } } main := &B{ @@ -290,7 +306,7 @@ func runBenchmarks(benchmarks []InternalBenchmark) bool { }, benchTime: benchTime, benchFunc: func(b *B) { - for _, Benchmark := range benchmarks { + for _, Benchmark := range bs { b.Run(Benchmark.Name, Benchmark.F) } }, @@ -328,19 +344,28 @@ func (b *B) processBench(ctx *benchContext) { // A subbenchmark is like any other benchmark. A benchmark that calls Run at // least once will not be measured itself and will be called once with N=1. func (b *B) Run(name string, f func(b *B)) bool { - if b.level > 0 { - name = b.name + "/" + rewrite(name) + benchName, ok, partial := b.name, true, false + if b.context != nil { + benchName, ok, partial = b.context.match.fullName(&b.common, name) + } + if !ok { + return true } b.hasSub = true sub := &B{ common: common{ - name: name, + name: benchName, level: b.level + 1, }, benchFunc: f, benchTime: b.benchTime, context: b.context, } + if partial { + // Partial name match, like -bench=X/Y matching BenchmarkX. + // Only process sub-benchmarks, if any. + sub.hasSub = true + } if sub.run1() { sub.run() } diff --git a/src/testing/match.go b/src/testing/match.go index 26b1f904..b18c6e7f 100644 --- a/src/testing/match.go +++ b/src/testing/match.go @@ -5,9 +5,131 @@ package testing import ( + "fmt" + "os" "strconv" + "strings" + "sync" ) +// matcher sanitizes, uniques, and filters names of subtests and subbenchmarks. +type matcher struct { + filter []string + matchFunc func(pat, str string) (bool, error) + + mu sync.Mutex + subNames map[string]int64 +} + +// TODO: fix test_main to avoid race and improve caching, also allowing to +// eliminate this Mutex. +var matchMutex sync.Mutex + +func newMatcher(matchString func(pat, str string) (bool, error), patterns, name string) *matcher { + var filter []string + if patterns != "" { + filter = splitRegexp(patterns) + for i, s := range filter { + filter[i] = rewrite(s) + } + // Verify filters before doing any processing. + for i, s := range filter { + if _, err := matchString(s, "non-empty"); err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for element %d of %s (%q): %s\n", i, name, s, err) + os.Exit(1) + } + } + } + return &matcher{ + filter: filter, + matchFunc: matchString, + subNames: map[string]int64{}, + } +} + +func (m *matcher) fullName(c *common, subname string) (name string, ok, partial bool) { + name = subname + + m.mu.Lock() + defer m.mu.Unlock() + + if c != nil && c.level > 0 { + name = m.unique(c.name, rewrite(subname)) + } + + matchMutex.Lock() + defer matchMutex.Unlock() + + // We check the full array of paths each time to allow for the case that + // a pattern contains a '/'. + elem := strings.Split(name, "/") + for i, s := range elem { + if i >= len(m.filter) { + break + } + if ok, _ := m.matchFunc(m.filter[i], s); !ok { + return name, false, false + } + } + return name, true, len(elem) < len(m.filter) +} + +func splitRegexp(s string) []string { + a := make([]string, 0, strings.Count(s, "/")) + cs := 0 + cp := 0 + for i := 0; i < len(s); { + switch s[i] { + case '[': + cs++ + case ']': + if cs--; cs < 0 { // An unmatched ']' is legal. + cs = 0 + } + case '(': + if cs == 0 { + cp++ + } + case ')': + if cs == 0 { + cp-- + } + case '\\': + i++ + case '/': + if cs == 0 && cp == 0 { + a = append(a, s[:i]) + s = s[i+1:] + i = 0 + continue + } + } + i++ + } + return append(a, s) +} + +// unique creates a unique name for the given parent and subname by affixing it +// with one or more counts, if necessary. +func (m *matcher) unique(parent, subname string) string { + name := fmt.Sprintf("%s/%s", parent, subname) + empty := subname == "" + for { + next, exists := m.subNames[name] + if !empty && !exists { + m.subNames[name] = 1 // next count is 1 + return name + } + // Name was already used. We increment with the count and append a + // string with the count. + m.subNames[name] = next + 1 + + // Add a count to guarantee uniqueness. + name = fmt.Sprintf("%s#%02d", name, next) + empty = false + } +} + // rewrite rewrites a subname to having only printable characters and no white // space. func rewrite(s string) string { diff --git a/src/testing/match_test.go b/src/testing/match_test.go new file mode 100644 index 00000000..8c09dc66 --- /dev/null +++ b/src/testing/match_test.go @@ -0,0 +1,186 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "reflect" + "regexp" + "unicode" +) + +// Verify that our IsSpace agrees with unicode.IsSpace. +func TestIsSpace(t *T) { + n := 0 + for r := rune(0); r <= unicode.MaxRune; r++ { + if isSpace(r) != unicode.IsSpace(r) { + t.Errorf("IsSpace(%U)=%t incorrect", r, isSpace(r)) + n++ + if n > 10 { + return + } + } + } +} + +func TestSplitRegexp(t *T) { + res := func(s ...string) []string { return s } + testCases := []struct { + pattern string + result []string + }{ + // Correct patterns + // If a regexp pattern is correct, all split regexps need to be correct + // as well. + {"", res("")}, + {"/", res("", "")}, + {"//", res("", "", "")}, + {"A", res("A")}, + {"A/B", res("A", "B")}, + {"A/B/", res("A", "B", "")}, + {"/A/B/", res("", "A", "B", "")}, + {"[A]/(B)", res("[A]", "(B)")}, + {"[/]/[/]", res("[/]", "[/]")}, + {"[/]/[:/]", res("[/]", "[:/]")}, + {"/]", res("", "]")}, + {"]/", res("]", "")}, + {"]/[/]", res("]", "[/]")}, + {`([)/][(])`, res(`([)/][(])`)}, + {"[(]/[)]", res("[(]", "[)]")}, + + // Faulty patterns + // Errors in original should produce at least one faulty regexp in results. + {")/", res(")/")}, + {")/(/)", res(")/(", ")")}, + {"a[/)b", res("a[/)b")}, + {"(/]", res("(/]")}, + {"(/", res("(/")}, + {"[/]/[/", res("[/]", "[/")}, + {`\p{/}`, res(`\p{`, "}")}, + {`\p/`, res(`\p`, "")}, + {`[[:/:]]`, res(`[[:/:]]`)}, + } + for _, tc := range testCases { + a := splitRegexp(tc.pattern) + if !reflect.DeepEqual(a, tc.result) { + t.Errorf("splitRegexp(%q) = %#v; want %#v", tc.pattern, a, tc.result) + } + + // If there is any error in the pattern, one of the returned subpatterns + // needs to have an error as well. + if _, err := regexp.Compile(tc.pattern); err != nil { + ok := true + for _, re := range a { + if _, err := regexp.Compile(re); err != nil { + ok = false + } + } + if ok { + t.Errorf("%s: expected error in any of %q", tc.pattern, a) + } + } + } +} + +func TestMatcher(t *T) { + testCases := []struct { + pattern string + parent, sub string + ok bool + partial bool + }{ + // Behavior without subtests. + {"", "", "TestFoo", true, false}, + {"TestFoo", "", "TestFoo", true, false}, + {"TestFoo/", "", "TestFoo", true, true}, + {"TestFoo/bar/baz", "", "TestFoo", true, true}, + {"TestFoo", "", "TestBar", false, false}, + {"TestFoo/", "", "TestBar", false, false}, + {"TestFoo/bar/baz", "", "TestBar/bar/baz", false, false}, + + // with subtests + {"", "TestFoo", "x", true, false}, + {"TestFoo", "TestFoo", "x", true, false}, + {"TestFoo/", "TestFoo", "x", true, false}, + {"TestFoo/bar/baz", "TestFoo", "bar", true, true}, + // Subtest with a '/' in its name still allows for copy and pasted names + // to match. + {"TestFoo/bar/baz", "TestFoo", "bar/baz", true, false}, + {"TestFoo/bar/baz", "TestFoo/bar", "baz", true, false}, + {"TestFoo/bar/baz", "TestFoo", "x", false, false}, + {"TestFoo", "TestBar", "x", false, false}, + {"TestFoo/", "TestBar", "x", false, false}, + {"TestFoo/bar/baz", "TestBar", "x/bar/baz", false, false}, + + // subtests only + {"", "TestFoo", "x", true, false}, + {"/", "TestFoo", "x", true, false}, + {"./", "TestFoo", "x", true, false}, + {"./.", "TestFoo", "x", true, false}, + {"/bar/baz", "TestFoo", "bar", true, true}, + {"/bar/baz", "TestFoo", "bar/baz", true, false}, + {"//baz", "TestFoo", "bar/baz", true, false}, + {"//", "TestFoo", "bar/baz", true, false}, + {"/bar/baz", "TestFoo/bar", "baz", true, false}, + {"//foo", "TestFoo", "bar/baz", false, false}, + {"/bar/baz", "TestFoo", "x", false, false}, + {"/bar/baz", "TestBar", "x/bar/baz", false, false}, + } + + for _, tc := range testCases { + m := newMatcher(regexp.MatchString, tc.pattern, "-test.run") + + parent := &common{name: tc.parent} + if tc.parent != "" { + parent.level = 1 + } + if n, ok, partial := m.fullName(parent, tc.sub); ok != tc.ok || partial != tc.partial { + t.Errorf("for pattern %q, fullName(parent=%q, sub=%q) = %q, ok %v partial %v; want ok %v partial %v", + tc.pattern, tc.parent, tc.sub, n, ok, partial, tc.ok, tc.partial) + } + } +} + +func TestNaming(t *T) { + m := newMatcher(regexp.MatchString, "", "") + + parent := &common{name: "x", level: 1} // top-level test. + + // Rig the matcher with some preloaded values. + m.subNames["x/b"] = 1000 + + testCases := []struct { + name, want string + }{ + // Uniqueness + {"", "x/#00"}, + {"", "x/#01"}, + + {"t", "x/t"}, + {"t", "x/t#01"}, + {"t", "x/t#02"}, + + {"a#01", "x/a#01"}, // user has subtest with this name. + {"a", "x/a"}, // doesn't conflict with this name. + {"a", "x/a#01#01"}, // conflict, add disambiguating string. + {"a", "x/a#02"}, // This string is claimed now, so resume + {"a", "x/a#03"}, // with counting. + {"a#02", "x/a#02#01"}, + + {"b", "x/b#1000"}, // rigged, see above + {"b", "x/b#1001"}, + + // // Sanitizing + {"A:1 B:2", "x/A:1_B:2"}, + {"s\t\r\u00a0", "x/s___"}, + {"\x01", `x/\x01`}, + {"\U0010ffff", `x/\U0010ffff`}, + } + + for i, tc := range testCases { + if got, _, _ := m.fullName(parent, tc.name); got != tc.want { + t.Errorf("%d:%s: got %q; want %q", i, tc.name, got, tc.want) + } + } +} diff --git a/src/testing/testing.go b/src/testing/testing.go index b3ca8b30..fc4e7fdf 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -19,10 +19,9 @@ import ( // Testing flags. var ( - flagVerbose bool - flagShort bool - flagRunRegexp string - flagBenchRegexp string + flagVerbose bool + flagShort bool + flagRunRegexp string ) var initRan bool @@ -37,7 +36,8 @@ func Init() { flag.BoolVar(&flagVerbose, "test.v", false, "verbose: print additional output") flag.BoolVar(&flagShort, "test.short", false, "short: run smaller test suite to save time") flag.StringVar(&flagRunRegexp, "test.run", "", "run: regexp of tests to run") - flag.StringVar(&flagBenchRegexp, "test.bench", "", "run: regexp of benchmarks to run") + + initBenchmarkFlags() } // common holds the elements common between T and B and @@ -318,31 +318,6 @@ func (m *M) Run() int { m.Tests = filtered } - if flagBenchRegexp != "" { - var filtered []InternalBenchmark - - // pre-test the regexp; we don't want to bother logging one failure for every test name if the regexp is broken - if _, err := m.deps.MatchString(flagBenchRegexp, "some-test-name"); err != nil { - fmt.Println("testing: invalid regexp for -test.bench:", err.Error()) - failures++ - } - - // filter the list of tests before we try to run them - for _, test := range m.Benchmarks { - // ignore the error; we already tested that the regexp compiles fine above - if match, _ := m.deps.MatchString(flagBenchRegexp, test.Name); match { - filtered = append(filtered, test) - } - } - - m.Benchmarks = filtered - flagVerbose = true - if flagRunRegexp == "" { - m.Tests = []InternalTest{} - } - } else { - m.Benchmarks = []InternalBenchmark{} - } if len(m.Tests) == 0 && len(m.Benchmarks) == 0 { fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") @@ -363,9 +338,10 @@ func (m *M) Run() int { } } - runBenchmarks(m.Benchmarks) - - if failures > 0 { + if len(m.Tests) == 0 && *matchBenchmarks == "" { + fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + } + if failures > 0 || !runBenchmarks(m.deps.MatchString, m.Benchmarks) { fmt.Println("FAIL") } else { if flagVerbose {