src/testing: add support for -benchmem

corpus_test.go

@@ -116,7 +116,7 @@ func TestCorpus(t *testing.T) {
 				tags.Set(repo.Tags)
 				opts.Tags = []string(tags)
 
-				passed, err := Test(path, out, out, &opts, false, testing.Verbose(), false, "", "", "", "")
+				passed, err := Test(path, out, out, &opts, false, testing.Verbose(), false, "", "", "", false, "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}

main.go

@@ -195,7 +195,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error {
 
 // Test runs the tests in the given package. Returns whether the test passed and
 // possibly an error if the test failed to run.
-func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string, outpath string) (bool, error) {
+func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string, testBenchMem bool, outpath string) (bool, error) {
 	options.TestConfig.CompileTestBinary = true
 	config, err := builder.NewConfig(options)
 	if err != nil {
@@ -219,6 +219,9 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 	if testBenchTime != "" {
 		flags = append(flags, "-test.benchtime="+testBenchTime)
 	}
+	if testBenchMem {
+		flags = append(flags, "-test.benchmem")
+	}
 
 	passed := false
 	err = buildAndRun(pkgName, config, os.Stdout, flags, nil, 0, func(cmd *exec.Cmd, result builder.BuildResult) error {
@@ -1333,6 +1336,7 @@ func main() {
 	var testBenchRegexp *string
 	var testBenchTime *string
 	var testRunRegexp *string
+	var testBenchMem *bool
 	if command == "help" || command == "test" {
 		testCompileOnlyFlag = flag.Bool("c", false, "compile the test binary but do not run it")
 		testVerboseFlag = flag.Bool("v", false, "verbose: print additional output")
@@ -1340,6 +1344,7 @@ func main() {
 		testRunRegexp = flag.String("run", "", "run: regexp of tests to run")
 		testBenchRegexp = flag.String("bench", "", "run: regexp of benchmarks to run")
 		testBenchTime = flag.String("benchtime", "", "run each benchmark for duration `d`")
+		testBenchMem = flag.Bool("benchmem", false, "show memory stats for benchmarks")
 	}
 
 	// Early command processing, before commands are interpreted by the Go flag
@@ -1570,7 +1575,7 @@ func main() {
 				defer close(buf.done)
 				stdout := (*testStdout)(buf)
 				stderr := (*testStderr)(buf)
-				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, *testBenchTime, outpath)
+				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, *testBenchTime, *testBenchMem, outpath)
 				if err != nil {
 					printCompilerError(func(args ...interface{}) {
 						fmt.Fprintln(stderr, args...)

main_test.go

@@ -432,7 +432,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "", false, "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -453,7 +453,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "", false, "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -480,7 +480,7 @@ func TestTest(t *testing.T) {
 
 				var output bytes.Buffer
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "", false, "")
 				if err != nil {
 					t.Errorf("test error: %v", err)
 				}
@@ -504,7 +504,7 @@ func TestTest(t *testing.T) {
 				defer out.Close()
 
 				opts := targ.opts
-				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "", "")
+				passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "", false, "")
 				if err == nil {
 					t.Error("test did not error")
 				}

src/testing/benchmark.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -18,11 +19,13 @@ import (
 
 func initBenchmarkFlags() {
 	matchBenchmarks = flag.String("test.bench", "", "run only benchmarks matching `regexp`")
+	benchmarkMemory = flag.Bool("test.benchmem", false, "print memory allocations for benchmarks")
 	flag.Var(&benchTime, "test.benchtime", "run each benchmark for duration `d`")
 }
 
 var (
 	matchBenchmarks *string
+	benchmarkMemory *bool
 	benchTime       = benchTimeFlag{d: 1 * time.Second} // changed during test of testing package
 )
 
@@ -85,6 +88,15 @@ type B struct {
 	benchTime    benchTimeFlag
 	timerOn      bool
 	result       BenchmarkResult
+
+	// report memory statistics
+	showAllocResult bool
+	// initial state of MemStats.Mallocs and MemStats.TotalAlloc
+	startAllocs uint64
+	startBytes  uint64
+	// net total after running benchmar
+	netAllocs uint64
+	netBytes  uint64
 }
 
 // StartTimer starts timing a test. This function is called automatically
@@ -94,6 +106,11 @@ func (b *B) StartTimer() {
 	if !b.timerOn {
 		b.start = time.Now()
 		b.timerOn = true
+
+		var mstats runtime.MemStats
+		runtime.ReadMemStats(&mstats)
+		b.startAllocs = mstats.Mallocs
+		b.startBytes = mstats.TotalAlloc
 	}
 }
 
@@ -104,6 +121,11 @@ func (b *B) StopTimer() {
 	if b.timerOn {
 		b.duration += time.Since(b.start)
 		b.timerOn = false
+
+		var mstats runtime.MemStats
+		runtime.ReadMemStats(&mstats)
+		b.netAllocs += mstats.Mallocs - b.startAllocs
+		b.netBytes += mstats.TotalAlloc - b.startBytes
 	}
 }
 
@@ -112,8 +134,15 @@ func (b *B) StopTimer() {
 func (b *B) ResetTimer() {
 	if b.timerOn {
 		b.start = time.Now()
+
+		var mstats runtime.MemStats
+		runtime.ReadMemStats(&mstats)
+		b.startAllocs = mstats.Mallocs
+		b.startBytes = mstats.TotalAlloc
 	}
 	b.duration = 0
+	b.netAllocs = 0
+	b.netBytes = 0
 }
 
 // SetBytes records the number of bytes processed in a single operation.
@@ -124,7 +153,7 @@ func (b *B) SetBytes(n int64) { b.bytes = n }
 // It is equivalent to setting -test.benchmem, but it only affects the
 // benchmark function that calls ReportAllocs.
 func (b *B) ReportAllocs() {
-	return // TODO: implement
+	b.showAllocResult = true
 }
 
 // runN runs a single benchmark for the specified number of iterations.
@@ -216,7 +245,7 @@ func (b *B) launch() {
 			b.runN(int(n))
 		}
 	}
-	b.result = BenchmarkResult{b.N, b.duration, b.bytes}
+	b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes}
 }
 
 // BenchmarkResult contains the results of a benchmark run.
@@ -224,6 +253,9 @@ type BenchmarkResult struct {
 	N     int           // The number of iterations.
 	T     time.Duration // The total time taken.
 	Bytes int64         // Bytes processed in one iteration.
+
+	MemAllocs uint64 // The total number of memory allocations.
+	MemBytes  uint64 // The total number of bytes allocated.
 }
 
 // NsPerOp returns the "ns/op" metric.
@@ -245,13 +277,19 @@ func (r BenchmarkResult) mbPerSec() float64 {
 // AllocsPerOp returns the "allocs/op" metric,
 // which is calculated as r.MemAllocs / r.N.
 func (r BenchmarkResult) AllocsPerOp() int64 {
-	return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go
+	if r.N <= 0 {
+		return 0
+	}
+	return int64(r.MemAllocs) / int64(r.N)
 }
 
 // AllocedBytesPerOp returns the "B/op" metric,
 // which is calculated as r.MemBytes / r.N.
 func (r BenchmarkResult) AllocedBytesPerOp() int64 {
-	return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go
+	if r.N <= 0 {
+		return 0
+	}
+	return int64(r.MemBytes) / int64(r.N)
 }
 
 // String returns a summary of the benchmark results.
@@ -278,6 +316,12 @@ func (r BenchmarkResult) String() string {
 	return buf.String()
 }
 
+// MemString returns r.AllocedBytesPerOp and r.AllocsPerOp in the same format as 'go test'.
+func (r BenchmarkResult) MemString() string {
+	return fmt.Sprintf("%8d B/op\t%8d allocs/op",
+		r.AllocedBytesPerOp(), r.AllocsPerOp())
+}
+
 func prettyPrint(w io.Writer, x float64, unit string) {
 	// Print all numbers with 10 places before the decimal point
 	// and small numbers with four sig figs. Field widths are
@@ -363,6 +407,10 @@ func (b *B) processBench(ctx *benchContext) {
 	}
 	if ctx != nil {
 		results := r.String()
+
+		if *benchmarkMemory || b.showAllocResult {
+			results += "\t" + r.MemString()
+		}
 		fmt.Println(results)
 	}
 }