From 21c76c0cb0cedadece76a2626c61c951b8d1681d Mon Sep 17 00:00:00 2001 From: Dan Kegel Date: Wed, 12 Jan 2022 08:36:46 -0800 Subject: [PATCH] testing: benchmarks: implement -benchtime flag --- main.go | 13 +++++-- main_test.go | 8 ++-- src/testing/benchmark.go | 82 ++++++++++++++++++++++++++++------------ 3 files changed, 70 insertions(+), 33 deletions(-) diff --git a/main.go b/main.go index eb041f83..52856bb7 100644 --- a/main.go +++ b/main.go @@ -179,7 +179,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error { // Test runs the tests in the given package. Returns whether the test passed and // possibly an error if the test failed to run. -func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, outpath string) (bool, error) { +func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string, outpath string) (bool, error) { options.TestConfig.CompileTestBinary = true config, err := builder.NewConfig(options) if err != nil { @@ -210,7 +210,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options }() start := time.Now() var err error - passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp, testBenchRegexp) + passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp, testBenchRegexp, testBenchTime) if err != nil { return err } @@ -236,7 +236,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options // runPackageTest runs a test binary that was previously built. The return // values are whether the test passed and any errors encountered while trying to // run the binary. -func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string) (bool, error) { +func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string) (bool, error) { var cmd *exec.Cmd if len(config.Target.Emulator) == 0 { // Run directly. @@ -253,6 +253,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result if testBenchRegexp != "" { flags = append(flags, "-test.bench="+testBenchRegexp) } + if testBenchTime != "" { + flags = append(flags, "-test.benchtime="+testBenchTime) + } cmd = executeCommand(config.Options, result.Binary, flags...) } else { // Run in an emulator. @@ -1207,6 +1210,7 @@ func main() { } var testCompileOnlyFlag, testVerboseFlag, testShortFlag *bool var testBenchRegexp *string + var testBenchTime *string var testRunRegexp *string if command == "help" || command == "test" { testCompileOnlyFlag = flag.Bool("c", false, "compile the test binary but do not run it") @@ -1214,6 +1218,7 @@ func main() { testShortFlag = flag.Bool("short", false, "short: run smaller test suite to save time") testRunRegexp = flag.String("run", "", "run: regexp of tests to run") testBenchRegexp = flag.String("bench", "", "run: regexp of benchmarks to run") + testBenchTime = flag.String("benchtime", "", "run each benchmark for duration `d`") } // Early command processing, before commands are interpreted by the Go flag @@ -1437,7 +1442,7 @@ func main() { defer close(buf.done) stdout := (*testStdout)(buf) stderr := (*testStderr)(buf) - passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, outpath) + passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, *testBenchTime, outpath) if err != nil { printCompilerError(func(args ...interface{}) { fmt.Fprintln(stderr, args...) diff --git a/main_test.go b/main_test.go index 5cd4fefd..0f6db9c7 100644 --- a/main_test.go +++ b/main_test.go @@ -514,7 +514,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "", "") if err != nil { t.Errorf("test error: %v", err) } @@ -535,7 +535,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "", "") if err != nil { t.Errorf("test error: %v", err) } @@ -562,7 +562,7 @@ func TestTest(t *testing.T) { var output bytes.Buffer opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "", "") if err != nil { t.Errorf("test error: %v", err) } @@ -586,7 +586,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "", "") if err == nil { t.Error("test did not error") } diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index ba1bfd30..caf38c35 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -11,12 +11,14 @@ import ( "fmt" "io" "math" + "strconv" "strings" "time" ) func initBenchmarkFlags() { matchBenchmarks = flag.String("test.bench", "", "run only benchmarks matching `regexp`") + flag.Var(&benchTime, "test.benchtime", "run each benchmark for duration `d`") } var ( @@ -26,6 +28,31 @@ var ( type benchTimeFlag struct { d time.Duration + n int +} + +func (f *benchTimeFlag) String() string { + if f.n > 0 { + return fmt.Sprintf("%dx", f.n) + } + return time.Duration(f.d).String() +} + +func (f *benchTimeFlag) Set(s string) error { + if strings.HasSuffix(s, "x") { + n, err := strconv.ParseInt(s[:len(s)-1], 10, 0) + if err != nil || n <= 0 { + return fmt.Errorf("invalid count") + } + *f = benchTimeFlag{n: int(n)} + return nil + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return fmt.Errorf("invalid duration") + } + *f = benchTimeFlag{d: d} + return nil } // InternalBenchmark is an internal type but exported because it is cross-package; @@ -160,32 +187,37 @@ func (b *B) doBench() BenchmarkResult { // of benchmark iterations until the benchmark runs for the requested benchtime. // run1 must have been called on b. func (b *B) launch() { - d := b.benchTime.d - for n := int64(1); !b.failed && b.duration < d && n < 1e9; { - last := n - // Predict required iterations. - goalns := d.Nanoseconds() - prevIters := int64(b.N) - prevns := b.duration.Nanoseconds() - if prevns <= 0 { - // Round up, to avoid div by zero. - prevns = 1 + // Run the benchmark for at least the specified amount of time. + if b.benchTime.n > 0 { + b.runN(b.benchTime.n) + } else { + d := b.benchTime.d + for n := int64(1); !b.failed && b.duration < d && n < 1e9; { + last := n + // Predict required iterations. + goalns := d.Nanoseconds() + prevIters := int64(b.N) + prevns := b.duration.Nanoseconds() + if prevns <= 0 { + // Round up, to avoid div by zero. + prevns = 1 + } + // Order of operations matters. + // For very fast benchmarks, prevIters ~= prevns. + // If you divide first, you get 0 or 1, + // which can hide an order of magnitude in execution time. + // So multiply first, then divide. + n = goalns * prevIters / prevns + // Run more iterations than we think we'll need (1.2x). + n += n / 5 + // Don't grow too fast in case we had timing errors previously. + n = min(n, 100*last) + // Be sure to run at least one more than last time. + n = max(n, last+1) + // Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.) + n = min(n, 1e9) + b.runN(int(n)) } - // Order of operations matters. - // For very fast benchmarks, prevIters ~= prevns. - // If you divide first, you get 0 or 1, - // which can hide an order of magnitude in execution time. - // So multiply first, then divide. - n = goalns * prevIters / prevns - // Run more iterations than we think we'll need (1.2x). - n += n / 5 - // Don't grow too fast in case we had timing errors previously. - n = min(n, 100*last) - // Be sure to run at least one more than last time. - n = max(n, last+1) - // Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.) - n = min(n, 1e9) - b.runN(int(n)) } b.result = BenchmarkResult{b.N, b.duration, b.bytes} }