From c4d99e5297b52fca4491717561b62fa43d222fe6 Mon Sep 17 00:00:00 2001 From: Damian Gryski Date: Wed, 10 Aug 2022 00:13:01 -0700 Subject: [PATCH] src/testing: add support for -benchmem --- corpus_test.go | 2 +- main.go | 9 +++++-- main_test.go | 8 +++--- src/testing/benchmark.go | 56 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/corpus_test.go b/corpus_test.go index 1f8c5bf9..ceb63169 100644 --- a/corpus_test.go +++ b/corpus_test.go @@ -116,7 +116,7 @@ func TestCorpus(t *testing.T) { tags.Set(repo.Tags) opts.Tags = []string(tags) - passed, err := Test(path, out, out, &opts, false, testing.Verbose(), false, "", "", "", "") + passed, err := Test(path, out, out, &opts, false, testing.Verbose(), false, "", "", "", false, "") if err != nil { t.Errorf("test error: %v", err) } diff --git a/main.go b/main.go index 1ae65641..e3b6d644 100644 --- a/main.go +++ b/main.go @@ -195,7 +195,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error { // Test runs the tests in the given package. Returns whether the test passed and // possibly an error if the test failed to run. -func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string, outpath string) (bool, error) { +func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, testBenchRegexp string, testBenchTime string, testBenchMem bool, outpath string) (bool, error) { options.TestConfig.CompileTestBinary = true config, err := builder.NewConfig(options) if err != nil { @@ -219,6 +219,9 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options if testBenchTime != "" { flags = append(flags, "-test.benchtime="+testBenchTime) } + if testBenchMem { + flags = append(flags, "-test.benchmem") + } passed := false err = buildAndRun(pkgName, config, os.Stdout, flags, nil, 0, func(cmd *exec.Cmd, result builder.BuildResult) error { @@ -1333,6 +1336,7 @@ func main() { var testBenchRegexp *string var testBenchTime *string var testRunRegexp *string + var testBenchMem *bool if command == "help" || command == "test" { testCompileOnlyFlag = flag.Bool("c", false, "compile the test binary but do not run it") testVerboseFlag = flag.Bool("v", false, "verbose: print additional output") @@ -1340,6 +1344,7 @@ func main() { testRunRegexp = flag.String("run", "", "run: regexp of tests to run") testBenchRegexp = flag.String("bench", "", "run: regexp of benchmarks to run") testBenchTime = flag.String("benchtime", "", "run each benchmark for duration `d`") + testBenchMem = flag.Bool("benchmem", false, "show memory stats for benchmarks") } // Early command processing, before commands are interpreted by the Go flag @@ -1570,7 +1575,7 @@ func main() { defer close(buf.done) stdout := (*testStdout)(buf) stderr := (*testStderr)(buf) - passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, *testBenchTime, outpath) + passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, *testBenchRegexp, *testBenchTime, *testBenchMem, outpath) if err != nil { printCompilerError(func(args ...interface{}) { fmt.Fprintln(stderr, args...) diff --git a/main_test.go b/main_test.go index bc790648..6bd79347 100644 --- a/main_test.go +++ b/main_test.go @@ -432,7 +432,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/pass", out, out, &opts, false, false, false, "", "", "", false, "") if err != nil { t.Errorf("test error: %v", err) } @@ -453,7 +453,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/fail", out, out, &opts, false, false, false, "", "", "", false, "") if err != nil { t.Errorf("test error: %v", err) } @@ -480,7 +480,7 @@ func TestTest(t *testing.T) { var output bytes.Buffer opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/nothing", io.MultiWriter(&output, out), out, &opts, false, false, false, "", "", "", false, "") if err != nil { t.Errorf("test error: %v", err) } @@ -504,7 +504,7 @@ func TestTest(t *testing.T) { defer out.Close() opts := targ.opts - passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "", "") + passed, err := Test("github.com/tinygo-org/tinygo/tests/testing/builderr", out, out, &opts, false, false, false, "", "", "", false, "") if err == nil { t.Error("test did not error") } diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index 80c6fdc1..5f106d99 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "math" + "runtime" "strconv" "strings" "time" @@ -18,11 +19,13 @@ import ( func initBenchmarkFlags() { matchBenchmarks = flag.String("test.bench", "", "run only benchmarks matching `regexp`") + benchmarkMemory = flag.Bool("test.benchmem", false, "print memory allocations for benchmarks") flag.Var(&benchTime, "test.benchtime", "run each benchmark for duration `d`") } var ( matchBenchmarks *string + benchmarkMemory *bool benchTime = benchTimeFlag{d: 1 * time.Second} // changed during test of testing package ) @@ -85,6 +88,15 @@ type B struct { benchTime benchTimeFlag timerOn bool result BenchmarkResult + + // report memory statistics + showAllocResult bool + // initial state of MemStats.Mallocs and MemStats.TotalAlloc + startAllocs uint64 + startBytes uint64 + // net total after running benchmar + netAllocs uint64 + netBytes uint64 } // StartTimer starts timing a test. This function is called automatically @@ -94,6 +106,11 @@ func (b *B) StartTimer() { if !b.timerOn { b.start = time.Now() b.timerOn = true + + var mstats runtime.MemStats + runtime.ReadMemStats(&mstats) + b.startAllocs = mstats.Mallocs + b.startBytes = mstats.TotalAlloc } } @@ -104,6 +121,11 @@ func (b *B) StopTimer() { if b.timerOn { b.duration += time.Since(b.start) b.timerOn = false + + var mstats runtime.MemStats + runtime.ReadMemStats(&mstats) + b.netAllocs += mstats.Mallocs - b.startAllocs + b.netBytes += mstats.TotalAlloc - b.startBytes } } @@ -112,8 +134,15 @@ func (b *B) StopTimer() { func (b *B) ResetTimer() { if b.timerOn { b.start = time.Now() + + var mstats runtime.MemStats + runtime.ReadMemStats(&mstats) + b.startAllocs = mstats.Mallocs + b.startBytes = mstats.TotalAlloc } b.duration = 0 + b.netAllocs = 0 + b.netBytes = 0 } // SetBytes records the number of bytes processed in a single operation. @@ -124,7 +153,7 @@ func (b *B) SetBytes(n int64) { b.bytes = n } // It is equivalent to setting -test.benchmem, but it only affects the // benchmark function that calls ReportAllocs. func (b *B) ReportAllocs() { - return // TODO: implement + b.showAllocResult = true } // runN runs a single benchmark for the specified number of iterations. @@ -216,7 +245,7 @@ func (b *B) launch() { b.runN(int(n)) } } - b.result = BenchmarkResult{b.N, b.duration, b.bytes} + b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes} } // BenchmarkResult contains the results of a benchmark run. @@ -224,6 +253,9 @@ type BenchmarkResult struct { N int // The number of iterations. T time.Duration // The total time taken. Bytes int64 // Bytes processed in one iteration. + + MemAllocs uint64 // The total number of memory allocations. + MemBytes uint64 // The total number of bytes allocated. } // NsPerOp returns the "ns/op" metric. @@ -245,13 +277,19 @@ func (r BenchmarkResult) mbPerSec() float64 { // AllocsPerOp returns the "allocs/op" metric, // which is calculated as r.MemAllocs / r.N. func (r BenchmarkResult) AllocsPerOp() int64 { - return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go + if r.N <= 0 { + return 0 + } + return int64(r.MemAllocs) / int64(r.N) } // AllocedBytesPerOp returns the "B/op" metric, // which is calculated as r.MemBytes / r.N. func (r BenchmarkResult) AllocedBytesPerOp() int64 { - return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go + if r.N <= 0 { + return 0 + } + return int64(r.MemBytes) / int64(r.N) } // String returns a summary of the benchmark results. @@ -278,6 +316,12 @@ func (r BenchmarkResult) String() string { return buf.String() } +// MemString returns r.AllocedBytesPerOp and r.AllocsPerOp in the same format as 'go test'. +func (r BenchmarkResult) MemString() string { + return fmt.Sprintf("%8d B/op\t%8d allocs/op", + r.AllocedBytesPerOp(), r.AllocsPerOp()) +} + func prettyPrint(w io.Writer, x float64, unit string) { // Print all numbers with 10 places before the decimal point // and small numbers with four sig figs. Field widths are @@ -363,6 +407,10 @@ func (b *B) processBench(ctx *benchContext) { } if ctx != nil { results := r.String() + + if *benchmarkMemory || b.showAllocResult { + results += "\t" + r.MemString() + } fmt.Println(results) } }