testing: benchmarks: implement -benchtime flag

main.go

@@ -179,7 +179,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error {
 
 // Test runs the tests in the given package. Returns whether the test passed and
 // possibly an error if the test failed to run.
-func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, outpath string) (bool, error) {
+func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string, outpath string) (bool, error) {
 	options.TestConfig.CompileTestBinary = true
 	config, err := builder.NewConfig(options)
 	if err != nil {
@@ -210,7 +210,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 		}()
 		start := time.Now()
 		var err error
-		passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp, testBenchRegexp)
+		passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp, testBenchRegexp, testBenchTime)
 		if err != nil {
 			return err
 		}
@@ -236,7 +236,7 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 // runPackageTest runs a test binary that was previously built. The return
 // values are whether the test passed and any errors encountered while trying to
 // run the binary.
-func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string) (bool, error) {
+func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string) (bool, error) {
 	var cmd *exec.Cmd
 	if len(config.Target.Emulator) == 0 {
 		// Run directly.
@@ -253,6 +253,9 @@ func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result
 		if testBenchRegexp != "" {
 			flags = append(flags, "-test.bench="+testBenchRegexp)
 		}
+		if testBenchTime != "" {
+			flags = append(flags, "-test.benchtime="+testBenchTime)
+		}
 		cmd = executeCommand(config.Options, result.Binary, flags...)
 	} else {
 		// Run in an emulator.
@@ -1207,6 +1210,7 @@ func main() {
 	}
 	var testCompileOnlyFlag, testVerboseFlag, testShortFlag *bool
 	var testBenchRegexp *string
+	var testBenchTime *string
 	var testRunRegexp *string
 	if command == "help" || command == "test" {
 		testCompileOnlyFlag = flag.Bool("c", false, "compile the test binary but do not run it")
@@ -1214,6 +1218,7 @@ func main() {
 		testShortFlag = flag.Bool("short", false, "short: run smaller test suite to save time")
 		testRunRegexp = flag.String("run", "", "run: regexp of tests to run")
 		testBenchRegexp = flag.String("bench", "", "run: regexp of benchmarks to run")
+		testBenchTime = flag.String("benchtime", "", "run each benchmark for duration `d`")
 	}
 
 	// Early command processing, before commands are interpreted by the Go flag
@@ -1437,7 +1442,7 @@ func main() {
 				defer close(buf.done)
 				stdout := (*testStdout)(buf)
 				stderr := (*testStderr)(buf)
-				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, outpath)
+				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, *testBenchTime, outpath)
 				if err != nil {
 					printCompilerError(func(args ...interface{}) {
 						fmt.Fprintln(stderr, args...)

main_test.go

@@ -514,7 +514,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -535,7 +535,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -562,7 +562,7 @@ func TestTest(t *testing.T) {
 
 				var output bytes.Buffer
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "", "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -586,7 +586,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "", "")
 				if err == nil {
 					t.Error("test did not error")
 				}

src/testing/benchmark.go

@@ -11,12 +11,14 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"strconv"
 	"strings"
 	"time"
 )
 
 func initBenchmarkFlags() {
 	matchBenchmarks = flag.String("test.bench", "", "run only benchmarks matching `regexp`")
+	flag.Var(&benchTime, "test.benchtime", "run each benchmark for duration `d`")
 }
 
 var (
@@ -26,6 +28,31 @@ var (
 
 type benchTimeFlag struct {
 	d time.Duration
+	n int
+}
+
+func (f *benchTimeFlag) String() string {
+	if f.n > 0 {
+		return fmt.Sprintf("%dx", f.n)
+	}
+	return time.Duration(f.d).String()
+}
+
+func (f *benchTimeFlag) Set(s string) error {
+	if strings.HasSuffix(s, "x") {
+		n, err := strconv.ParseInt(s[:len(s)-1], 10, 0)
+		if err != nil || n <= 0 {
+			return fmt.Errorf("invalid count")
+		}
+		*f = benchTimeFlag{n: int(n)}
+		return nil
+	}
+	d, err := time.ParseDuration(s)
+	if err != nil || d <= 0 {
+		return fmt.Errorf("invalid duration")
+	}
+	*f = benchTimeFlag{d: d}
+	return nil
 }
 
 // InternalBenchmark is an internal type but exported because it is cross-package;
@@ -160,32 +187,37 @@ func (b *B) doBench() BenchmarkResult {
 // of benchmark iterations until the benchmark runs for the requested benchtime.
 // run1 must have been called on b.
 func (b *B) launch() {
-	d := b.benchTime.d
+	// Run the benchmark for at least the specified amount of time.
-	for n := int64(1); !b.failed && b.duration < d && n < 1e9; {
+	if b.benchTime.n > 0 {
-		last := n
+		b.runN(b.benchTime.n)
-		// Predict required iterations.
+	} else {
-		goalns := d.Nanoseconds()
+		d := b.benchTime.d
-		prevIters := int64(b.N)
+		for n := int64(1); !b.failed && b.duration < d && n < 1e9; {
-		prevns := b.duration.Nanoseconds()
+			last := n
-		if prevns <= 0 {
+			// Predict required iterations.
-			// Round up, to avoid div by zero.
+			goalns := d.Nanoseconds()
-			prevns = 1
+			prevIters := int64(b.N)
+			prevns := b.duration.Nanoseconds()
+			if prevns <= 0 {
+				// Round up, to avoid div by zero.
+				prevns = 1
+			}
+			// Order of operations matters.
+			// For very fast benchmarks, prevIters ~= prevns.
+			// If you divide first, you get 0 or 1,
+			// which can hide an order of magnitude in execution time.
+			// So multiply first, then divide.
+			n = goalns * prevIters / prevns
+			// Run more iterations than we think we'll need (1.2x).
+			n += n / 5
+			// Don't grow too fast in case we had timing errors previously.
+			n = min(n, 100*last)
+			// Be sure to run at least one more than last time.
+			n = max(n, last+1)
+			// Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.)
+			n = min(n, 1e9)
+			b.runN(int(n))
 		}
-		// Order of operations matters.
-		// For very fast benchmarks, prevIters ~= prevns.
-		// If you divide first, you get 0 or 1,
-		// which can hide an order of magnitude in execution time.
-		// So multiply first, then divide.
-		n = goalns * prevIters / prevns
-		// Run more iterations than we think we'll need (1.2x).
-		n += n / 5
-		// Don't grow too fast in case we had timing errors previously.
-		n = min(n, 100*last)
-		// Be sure to run at least one more than last time.
-		n = max(n, last+1)
-		// Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.)
-		n = min(n, 1e9)
-		b.runN(int(n))
 	}
 	b.result = BenchmarkResult{b.N, b.duration, b.bytes}
 }