main (tinygo test): build and run tests in concurrently

main.go

@@ -18,6 +18,7 @@ import (
 	"runtime/pprof"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -177,7 +178,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error {
 
 // Test runs the tests in the given package. Returns whether the test passed and
 // possibly an error if the test failed to run.
-func Test(pkgName string, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, outpath string) (bool, error) {
+func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options, testCompileOnly, testVerbose, testShort bool, testRunRegexp string, outpath string) (bool, error) {
 	options.TestConfig.CompileTestBinary = true
 	config, err := builder.NewConfig(options)
 	if err != nil {
@@ -201,9 +202,13 @@ func Test(pkgName string, options *compileopts.Options, testCompileOnly, testVer
 		}
 
 		// Run the test.
+		config.Options.Semaphore <- struct{}{}
+		defer func() {
+			<-config.Options.Semaphore
+		}()
 		start := time.Now()
 		var err error
-		passed, err = runPackageTest(config, result, testVerbose, testShort, testRunRegexp)
+		passed, err = runPackageTest(config, stdout, stderr, result, testVerbose, testShort, testRunRegexp)
 		if err != nil {
 			return err
 		}
@@ -212,14 +217,14 @@ func Test(pkgName string, options *compileopts.Options, testCompileOnly, testVer
 		// Print the result.
 		importPath := strings.TrimSuffix(result.ImportPath, ".test")
 		if passed {
-			fmt.Printf("ok  \t%s\t%.3fs\n", importPath, duration.Seconds())
+			fmt.Fprintf(stdout, "ok  \t%s\t%.3fs\n", importPath, duration.Seconds())
 		} else {
-			fmt.Printf("FAIL\t%s\t%.3fs\n", importPath, duration.Seconds())
+			fmt.Fprintf(stdout, "FAIL\t%s\t%.3fs\n", importPath, duration.Seconds())
 		}
 		return nil
 	})
 	if err, ok := err.(loader.NoTestFilesError); ok {
-		fmt.Printf("?   \t%s\t[no test files]\n", err.ImportPath)
+		fmt.Fprintf(stdout, "?   \t%s\t[no test files]\n", err.ImportPath)
 		// Pretend the test passed - it at least didn't fail.
 		return true, nil
 	}
@@ -229,7 +234,7 @@ func Test(pkgName string, options *compileopts.Options, testCompileOnly, testVer
 // runPackageTest runs a test binary that was previously built. The return
 // values are whether the test passed and any errors encountered while trying to
 // run the binary.
-func runPackageTest(config *compileopts.Config, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string) (bool, error) {
+func runPackageTest(config *compileopts.Config, stdout, stderr io.Writer, result builder.BuildResult, testVerbose, testShort bool, testRunRegexp string) (bool, error) {
 	var cmd *exec.Cmd
 	if len(config.Target.Emulator) == 0 {
 		// Run directly.
@@ -264,8 +269,8 @@ func runPackageTest(config *compileopts.Config, result builder.BuildResult, test
 		cmd = executeCommand(config.Options, config.Target.Emulator[0], args...)
 	}
 	cmd.Dir = result.MainDir
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
 	err := cmd.Run()
 	if err != nil {
 		if _, ok := err.(*exec.ExitError); ok {
@@ -1368,16 +1373,57 @@ func main() {
 		if len(pkgNames) == 0 {
 			pkgNames = []string{"."}
 		}
-		allTestsPassed := true
-		for _, pkgName := range pkgNames {
-			// TODO: parallelize building the test binaries
-			passed, err := Test(pkgName, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, outpath)
-			handleCompilerError(err)
-			if !passed {
-				allTestsPassed = false
+		if *testCompileOnlyFlag && len(pkgNames) > 1 {
+			fmt.Println("cannot use -c flag with multiple packages")
+			os.Exit(1)
+		}
+
+		// Build and run the tests concurrently, buffering the output.
+		fail := make(chan struct{}, 1)
+		var wg sync.WaitGroup
+		bufs := make([]testOutputBuf, len(pkgNames))
+		for i, pkgName := range pkgNames {
+			pkgName := pkgName
+			buf := &bufs[i]
+			buf.done = make(chan struct{})
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				defer close(buf.done)
+				stdout := (*testStdout)(buf)
+				stderr := (*testStderr)(buf)
+				passed, err := Test(pkgName, stdout, stderr, options, *testCompileOnlyFlag, *testVerboseFlag, *testShortFlag, *testRunRegexp, outpath)
+				if err != nil {
+					printCompilerError(func(args ...interface{}) {
+						fmt.Fprintln(stderr, args...)
+					}, err)
+				}
+				if !passed {
+					select {
+					case fail <- struct{}{}:
+					default:
+					}
+				}
+			}()
+		}
+
+		// Flush the output one test at a time.
+		// This ensures that outputs from different tests are not mixed together.
+		for i := range bufs {
+			err := bufs[i].flush(os.Stdout, os.Stderr)
+			if err != nil {
+				// There was an error writing to stdout or stderr, so we probbably cannot print this.
+				select {
+				case fail <- struct{}{}:
+				default:
+				}
 			}
 		}
-		if !allTestsPassed {
+
+		// Wait for all tests to finish.
+		wg.Wait()
+		close(fail)
+		if _, fail := <-fail; fail {
 			fmt.Println("FAIL")
 			os.Exit(1)
 		}
@@ -1515,3 +1561,121 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// testOutputBuf is used to buffer the output of concurrent tests.
+type testOutputBuf struct {
+	mu             sync.Mutex
+	output         []outputEntry
+	stdout, stderr io.Writer
+	outerr, errerr error
+	done           chan struct{}
+}
+
+// flush the output to stdout and stderr.
+// This waits until done is closed.
+func (b *testOutputBuf) flush(stdout, stderr io.Writer) error {
+	b.mu.Lock()
+
+	var err error
+	b.stdout = stdout
+	b.stderr = stderr
+	for _, e := range b.output {
+		var w io.Writer
+		var errDst *error
+		if e.stderr {
+			w = stderr
+			errDst = &b.errerr
+		} else {
+			w = stdout
+			errDst = &b.outerr
+		}
+		if *errDst != nil {
+			continue
+		}
+
+		_, werr := w.Write(e.data)
+		if werr != nil {
+			if err == nil {
+				err = werr
+			}
+			*errDst = err
+		}
+	}
+
+	b.mu.Unlock()
+
+	<-b.done
+
+	return err
+}
+
+// testStdout writes stdout from a test to the output buffer.
+type testStdout testOutputBuf
+
+func (out *testStdout) Write(data []byte) (int, error) {
+	buf := (*testOutputBuf)(out)
+	buf.mu.Lock()
+
+	if buf.stdout != nil {
+		// Write the output directly.
+		err := out.outerr
+		buf.mu.Unlock()
+		if err != nil {
+			return 0, err
+		}
+		return buf.stdout.Write(data)
+	}
+
+	defer buf.mu.Unlock()
+
+	// Append the output.
+	var prev []byte
+	if len(buf.output) > 0 && !buf.output[len(buf.output)-1].stderr {
+		prev = buf.output[len(buf.output)-1].data
+		buf.output = buf.output[:len(buf.output)-1]
+	}
+	buf.output = append(buf.output, outputEntry{
+		stderr: false,
+		data:   append(prev, data...),
+	})
+
+	return len(data), nil
+}
+
+// testStderr writes stderr from a test to the output buffer.
+type testStderr testOutputBuf
+
+func (out *testStderr) Write(data []byte) (int, error) {
+	buf := (*testOutputBuf)(out)
+	buf.mu.Lock()
+
+	if buf.stderr != nil {
+		// Write the output directly.
+		err := out.errerr
+		buf.mu.Unlock()
+		if err != nil {
+			return 0, err
+		}
+		return buf.stderr.Write(data)
+	}
+
+	defer buf.mu.Unlock()
+
+	// Append the output.
+	var prev []byte
+	if len(buf.output) > 0 && buf.output[len(buf.output)-1].stderr {
+		prev = buf.output[len(buf.output)-1].data
+		buf.output = buf.output[:len(buf.output)-1]
+	}
+	buf.output = append(buf.output, outputEntry{
+		stderr: true,
+		data:   append(prev, data...),
+	})
+
+	return len(data), nil
+}
+
+type outputEntry struct {
+	stderr bool
+	data   []byte
+}