testing: --run now allows filtering of subtests

Also fix typo in error message in sub_test.go from upstream,
and move a few members from B to common where they belonged.

Note that testdata/testing.go seems to be pushing the edge of what
the emulated cortex-m3 target can handle; using regexp in that test
causes it to fail on that target with an out of memory error.

TODO: once tinygo supports runtime.Goexit, consider just using upstream's testing directory...

src/testing/benchmark.go

@@ -77,9 +77,6 @@ type InternalBenchmark struct {
 // affecting benchmark results.
 type B struct {
 	common
-	hasSub       bool          // TODO: should be in common, and atomic
-	start        time.Time     // TODO: should be in common
-	duration     time.Duration // TODO: should be in common
 	context      *benchContext
 	N            int
 	benchFunc    func(b *B)

src/testing/sub_test.go

@@ -32,7 +32,7 @@ func TestRunCleanup(t *T) {
 		t.Errorf("unexpected inner cleanup count; got %d want 1", innerCleanup)
 	}
 	if outerCleanup != 1 {
-		t.Errorf("unexpected outer cleanup count; got %d want 0", outerCleanup)
+		t.Errorf("unexpected outer cleanup count; got %d want 1", outerCleanup) // wrong upstream!
 	}
 }
 

src/testing/testing.go

@@ -12,7 +12,6 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"io"
 	"os"
 	"strings"
 	"time"
@@ -45,7 +44,6 @@ func Init() {
 // captures common methods such as Errorf.
 type common struct {
 	output   bytes.Buffer
-	w        io.Writer // either &output, or at top level, os.Stdout
 	indent   string
 	ran      bool     // Test or benchmark (or one of its subtests) was executed.
 	failed   bool     // Test or benchmark has failed.
@@ -53,6 +51,8 @@ type common struct {
 	cleanups []func() // optional functions to be called at the end of the test
 	finished bool     // Test function has completed.
 
+	hasSub bool // TODO: should be atomic
+
 	parent   *common
 	level    int       // Nesting depth of test or benchmark.
 	name     string    // Name of test or benchmark.
@@ -77,6 +77,19 @@ func Verbose() bool {
 	return flagVerbose
 }
 
+// flushToParent writes c.output to the parent after first writing the header
+// with the given format and arguments.
+func (c *common) flushToParent(testName, format string, args ...interface{}) {
+	if c.parent == nil {
+		// The fake top-level test doesn't want a FAIL or PASS banner.
+		// Not quite sure how this works upstream.
+		c.output.WriteTo(os.Stdout)
+	} else {
+		fmt.Fprintf(&c.parent.output, format, args...)
+		c.output.WriteTo(&c.parent.output)
+	}
+}
+
 // fmtDuration returns a string representing d in the form "87.00s".
 func fmtDuration(d time.Duration) string {
 	return fmt.Sprintf("%.2fs", d.Seconds())
@@ -110,6 +123,7 @@ var _ TB = (*B)(nil)
 //
 type T struct {
 	common
+	context *testContext // For running tests and subtests.
 }
 
 // Name returns the name of the running test or benchmark.
@@ -117,6 +131,13 @@ func (c *common) Name() string {
 	return c.name
 }
 
+func (c *common) setRan() {
+	if c.parent != nil {
+		c.parent.setRan()
+	}
+	c.ran = true
+}
+
 // Fail marks the function as having failed but continues execution.
 func (c *common) Fail() {
 	c.failed = true
@@ -270,35 +291,57 @@ func tRunner(t *T, fn func(t *T)) {
 	}()
 
 	// Run the test.
-	if flagVerbose {
-		fmt.Fprintf(t.w, "=== RUN   %s\n", t.name)
-	}
-
 	t.start = time.Now()
 	fn(t)
 	t.duration += time.Since(t.start) // TODO: capture cleanup time, too.
 
 	t.report() // Report after all subtests have finished.
-	t.ran = true
+	if t.parent != nil && !t.hasSub {
+		t.setRan()
+	}
 }
 
 // Run runs f as a subtest of t called name. It waits until the subtest is finished
 // and returns whether the subtest succeeded.
 func (t *T) Run(name string, f func(t *T)) bool {
+	t.hasSub = true
+	testName, ok, _ := t.context.match.fullName(&t.common, name)
+	if !ok {
+		return true
+	}
+
 	// Create a subtest.
 	sub := T{
 		common: common{
-			name:   t.name + "/" + rewrite(name),
-			indent: t.indent + "    ",
-			w:      &t.output,
+			name:   testName,
 			parent: &t.common,
+			level:  t.level + 1,
 		},
+		context: t.context,
+	}
+	if t.level > 0 {
+		sub.indent = sub.indent + "    "
+	}
+	if flagVerbose {
+		fmt.Fprintf(&t.output, "=== RUN   %s\n", sub.name)
 	}
 
 	tRunner(&sub, f)
 	return !sub.failed
 }
 
+// testContext holds all fields that are common to all tests. This includes
+// synchronization primitives to run at most *parallel tests.
+type testContext struct {
+	match *matcher
+}
+
+func newTestContext(m *matcher) *testContext {
+	return &testContext{
+		match: m,
+	}
+}
+
 // M is a test suite.
 type M struct {
 	// tests is a list of the test names to execute
@@ -353,40 +396,20 @@ func (m *M) Run() (code int) {
 
 func runTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ran, ok bool) {
 	ok = true
-	if flagRunRegexp != "" {
-		var filtered []InternalTest
 
-		// pre-test the regexp; we don't want to bother logging one failure for every test name if the regexp is broken
-		if _, err := matchString(flagRunRegexp, "some-test-name"); err != nil {
-			fmt.Println("testing: invalid regexp for -test.run:", err.Error())
-			return false, false
-		}
-
-		// filter the list of tests before we try to run them
-		for _, test := range tests {
-			// ignore the error; we already tested that the regexp compiles fine above
-			if match, _ := matchString(flagRunRegexp, test.Name); match {
-				filtered = append(filtered, test)
-			}
-		}
-
-		tests = filtered
-	}
-
-	for _, test := range tests {
+	ctx := newTestContext(newMatcher(matchString, flagRunRegexp, "-test.run"))
 	t := &T{
-			common: common{
-				name: test.Name,
-				w:    os.Stdout,
-			},
+		context: ctx,
 	}
 
-		tRunner(t, test.F)
-
+	tRunner(t, func(t *T) {
+		for _, test := range tests {
+			t.Run(test.Name, test.F)
 			ok = ok && !t.Failed()
-		ran = ran || t.ran
 		}
-	return ran, ok
+	})
+
+	return t.ran, ok
 }
 
 func (t *T) report() {
@@ -396,15 +419,13 @@ func (t *T) report() {
 		if t.parent != nil {
 			t.parent.failed = true
 		}
-		fmt.Fprintf(t.w, format, "FAIL", t.name, dstr)
-		t.w.Write(t.output.Bytes())
+		t.flushToParent(t.name, format, "FAIL", t.name, dstr)
 	} else if flagVerbose {
 		if t.Skipped() {
-			fmt.Fprintf(t.w, format, "SKIP", t.name, dstr)
+			t.flushToParent(t.name, format, "SKIP", t.name, dstr)
 		} else {
-			fmt.Fprintf(t.w, format, "PASS", t.name, dstr)
+			t.flushToParent(t.name, format, "PASS", t.name, dstr)
 		}
-		t.w.Write(t.output.Bytes())
 	}
 }
 

testdata/testing.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"errors"
+	"flag"
 	"io"
 	"testing"
 )
@@ -26,15 +27,60 @@ func TestBar(t *testing.T) {
 	t.Log("log Bar end")
 }
 
+func TestAllLowercase(t *testing.T) {
+	names := []string {
+		"alpha",
+		"BETA",
+		"gamma",
+		"DELTA",
+	}
+
+	for _, name := range names {
+		t.Run(name, func(t *testing.T) {
+			if 'a' <= name[0] && name[0] <= 'a' {
+				t.Logf("expected lowercase name, and got one, so I'm happy")
+			} else {
+				t.Errorf("expected lowercase name, got %s", name)
+			}
+		})
+	}
+}
+
 var tests = []testing.InternalTest{
 	{"TestFoo", TestFoo},
 	{"TestBar", TestBar},
+	{"TestAllLowercase", TestAllLowercase},
 }
 
 var benchmarks = []testing.InternalBenchmark{}
 
 var examples = []testing.InternalExample{}
 
+// A fake regexp matcher that can only handle two patterns.
+// Inflexible, but saves 50KB of flash and 50KB of RAM per -size full,
+// and lets tests pass on cortex-m3.
+func fakeMatchString(pat, str string) (bool, error) {
+	if pat == ".*" {
+		return true, nil
+	}
+	if pat == "[BD]" {
+		return (str[0] == 'B' || str[0] == 'D'), nil
+	}
+	println("BUG: fakeMatchString does not grok", pat)
+	return false, nil
+}
+
+func main() {
+	testing.Init()
+	flag.Set("test.run", ".*/[BD]")
+	m := testing.MainStart(matchStringOnly(fakeMatchString /*regexp.MatchString*/), tests, benchmarks, examples)
+
+	exitcode := m.Run()
+	if exitcode != 0 {
+		println("exitcode:", exitcode)
+	}
+}
+
 var errMain = errors.New("testing: unexpected use of func Main")
 
 // matchStringOnly is part of upstream, and is used below to provide a dummy deps to pass to MainStart
@@ -50,11 +96,3 @@ func (f matchStringOnly) ImportPath() string                          { return "
 func (f matchStringOnly) StartTestLog(io.Writer)                      {}
 func (f matchStringOnly) StopTestLog() error                          { return errMain }
 func (f matchStringOnly) SetPanicOnExit0(bool)                        {}
-
-func main() {
-	m := testing.MainStart(matchStringOnly(nil), tests, benchmarks, examples)
-	exitcode := m.Run()
-	if exitcode != 0 {
-		println("exitcode:", exitcode)
-	}
-}

testdata/testing.txt

@@ -12,5 +12,10 @@
         failed
         after failed
     log Bar end
+--- FAIL: TestAllLowercase (0.00s)
+    --- FAIL: TestAllLowercase/BETA (0.00s)
+        expected lowercase name, got BETA
+    --- FAIL: TestAllLowercase/DELTA (0.00s)
+        expected lowercase name, got DELTA
 FAIL
 exitcode: 1