From ee4e42ba1fea45a7a1516c7a9d7f93d32a514d15 Mon Sep 17 00:00:00 2001 From: Dan Kegel Date: Mon, 16 Aug 2021 08:45:28 -0700 Subject: [PATCH] src/testing: support -bench option to run benchmarks matching the given pattern. --- main.go | 16 ++++++++++---- main_test.go | 8 +++---- src/testing/benchmark.go | 26 ++++++++++++++++++++++- src/testing/testing.go | 45 +++++++++++++++++++++++++++++++++------- 4 files changed, 79 insertions(+), 16 deletions(-) diff --git a/main.go b/main.go index 80c96c93..b95c7e1e 100644 --- a/main.go +++ b/main.go @@ -178,7 +178,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error { // Test runs the tests in the given package. Returns whether the test passed and // possibly an error if the test failed to run. -func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, outpath string) (bool, error) { +func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, outpath string) (bool, error) { options.TestConfig.CompileTestBinary = true config, err := builder.NewConfig(options) if err != nil { @@ -209,7 +209,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options }() start := time.Now() var err error - passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp) + passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp, testBenchRegexp) if err != nil { return err } @@ -235,7 +235,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options // runPackageTest runs a test binary that was previously built. The return // values are whether the test passed and any errors encountered while trying to // run the binary. -func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string) (bool, error) { +func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string) (bool, error) { var cmd *exec.Cmd if len(config.Target.Emulator) == 0 { // Run directly. @@ -249,6 +249,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result if testRunRegexp != "" { flags = append(flags, "-test.run="+testRunRegexp) } + if testBenchRegexp != "" { + flags = append(flags, "-test.bench="+testBenchRegexp) + } cmd = executeCommand(config.Options, result.Binary, flags...) } else { // Run in an emulator. @@ -274,6 +277,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result if testRunRegexp != "" { args = append(args, "-test.run="+testRunRegexp) } + if testBenchRegexp != "" { + args = append(args, "-test.bench="+testBenchRegexp) + } } cmd = executeCommand(config.Options, config.Target.Emulator[0], args...) } @@ -1197,12 +1203,14 @@ func main() { flag.StringVar(&outpath, "o", "", "output filename") } var testCompileOnlyFlag, testVerboseFlag, testShortFlag *bool + var testBenchRegexp *string var testRunRegexp *string if command == "help" || command == "test" { testCompileOnlyFlag = flag.Bool("c", false, "compile the test binary but do not run it") testVerboseFlag = flag.Bool("v", false, "verbose: print additional output") testShortFlag = flag.Bool("short", false, "short: run smaller test suite to save time") testRunRegexp = flag.String("run", "", "run: regexp of tests to run") + testBenchRegexp = flag.String("bench", "", "run: regexp of benchmarks to run") } // Early command processing, before commands are interpreted by the Go flag @@ -1426,7 +1434,7 @@ func main() { defer close(buf.done) stdout := (*testStdout)(buf) stderr := (*testStderr)(buf) - passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, outpath) + passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, outpath) if err != nil { printCompilerError(func(args ...interface{}) { fmt.Fprintln(stderr, args...) diff --git a/main_test.go b/main_test.go index bb1e26f3..e8dbf7d1 100644 --- a/main_test.go +++ b/main_test.go @@ -516,7 +516,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "") if err != nil { t.Errorf("test error: %v", err) } @@ -537,7 +537,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "") if err != nil { t.Errorf("test error: %v", err) } @@ -564,7 +564,7 @@ func TestTest(t *testing.T) { var output bytes.Buffer opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "") if err != nil { t.Errorf("test error: %v", err) } @@ -588,7 +588,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "") if err == nil { t.Error("test did not error") } diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index 4647cfe9..88aec366 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -7,6 +7,7 @@ package testing import ( + "fmt" "time" ) @@ -187,6 +188,30 @@ func (r BenchmarkResult) AllocedBytesPerOp() int64 { return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go } +func runBenchmarks(benchmarks []InternalBenchmark) bool { + if len(benchmarks) == 0 { + return true + } + main := &B{ + common: common{ + name: "Main", + }, + benchTime: benchTime, + benchFunc: func(b *B) { + for _, Benchmark := range benchmarks { + if flagVerbose { + fmt.Printf("=== RUN %s\n", Benchmark.Name) + } + b.Run(Benchmark.Name, Benchmark.F) + fmt.Printf("--- Result: %d ns/op\n", b.result.NsPerOp()) + } + }, + } + + main.runN(1) + return true +} + // Run benchmarks f as a subbenchmark with the given name. It reports // true if the subbenchmark succeeded. // @@ -248,4 +273,3 @@ func Benchmark(f func(b *B)) BenchmarkResult { } return b.result } - diff --git a/src/testing/testing.go b/src/testing/testing.go index fe1fb83d..ee4056eb 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -18,9 +18,10 @@ import ( // Testing flags. var ( - flagVerbose bool - flagShort bool - flagRunRegexp string + flagVerbose bool + flagShort bool + flagRunRegexp string + flagBenchRegexp string ) var initRan bool @@ -35,6 +36,7 @@ func Init() { flag.BoolVar(&flagVerbose, "test.v", false, "verbose: print additional output") flag.BoolVar(&flagShort, "test.short", false, "short: run smaller test suite to save time") flag.StringVar(&flagRunRegexp, "test.run", "", "run: regexp of tests to run") + flag.StringVar(&flagBenchRegexp, "test.bench", "", "run: regexp of benchmarks to run") } // common holds the elements common between T and B and @@ -243,7 +245,8 @@ type InternalTest struct { // M is a test suite. type M struct { // tests is a list of the test names to execute - Tests []InternalTest + Tests []InternalTest + Benchmarks []InternalBenchmark deps testDeps } @@ -275,8 +278,33 @@ func (m *M) Run() int { m.Tests = filtered } + if flagBenchRegexp != "" { + var filtered []InternalBenchmark - if len(m.Tests) == 0 { + // pre-test the regexp; we don't want to bother logging one failure for every test name if the regexp is broken + if _, err := m.deps.MatchString(flagBenchRegexp, "some-test-name"); err != nil { + fmt.Println("testing: invalid regexp for -test.bench:", err.Error()) + failures++ + } + + // filter the list of tests before we try to run them + for _, test := range m.Benchmarks { + // ignore the error; we already tested that the regexp compiles fine above + if match, _ := m.deps.MatchString(flagBenchRegexp, test.Name); match { + filtered = append(filtered, test) + } + } + + m.Benchmarks = filtered + flagVerbose = true + if flagRunRegexp == "" { + m.Tests = []InternalTest{} + } + } else { + m.Benchmarks = []InternalBenchmark{} + } + + if len(m.Tests) == 0 && len(m.Benchmarks) == 0 { fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") } @@ -307,6 +335,8 @@ func (m *M) Run() int { } } + runBenchmarks(m.Benchmarks) + if failures > 0 { fmt.Println("FAIL") } else { @@ -358,8 +388,9 @@ type testDeps interface { func MainStart(deps interface{}, tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) *M { Init() return &M{ - Tests: tests, - deps: deps.(testDeps), + Tests: tests, + Benchmarks: benchmarks, + deps: deps.(testDeps), } }