loader: rewrite/refactor much of the code to use go list directly

There were a few problems with the go/packages package. While it is more
or less designed for our purpose, it didn't work quite well as it didn't
provide access to indirectly imported packages (most importantly the
runtime package). This led to a workaround that sometimes broke
`tinygo test`.

This PR contains a number of related changes:

  * It uses `go list` directly to retrieve the list of packages/files to
    compile, instead of relying on the go/packages package.
  * It replaces our custom TestMain replace code with the standard code
    for running tests (generated by `go list`).
  * It adds a dummy runtime/pprof package and modifies the testing
    package, to get tests to run again with the code generated by
    `go list`.

compiler/compiler.go

@@ -5,18 +5,15 @@ import (
 	"errors"
 	"fmt"
 	"go/ast"
-	"go/build"
 	"go/constant"
 	"go/token"
 	"go/types"
-	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/tinygo-org/tinygo/compileopts"
 	"github.com/tinygo-org/tinygo/compiler/llvmutil"
-	"github.com/tinygo-org/tinygo/goenv"
 	"github.com/tinygo-org/tinygo/ir"
 	"github.com/tinygo-org/tinygo/loader"
 	"golang.org/x/tools/go/ssa"
@@ -172,40 +169,12 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 	c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace()
 	dummyFunc.EraseFromParentAsFunction()
 
-	wd, err := os.Getwd()
+	lprogram, err := loader.Load(c.Config, []string{pkgName}, c.ClangHeaders, types.Config{
-	if err != nil {
-		return c.mod, nil, nil, []error{err}
-	}
-	goroot, err := loader.GetCachedGoroot(c.Config)
-	if err != nil {
-		return c.mod, nil, nil, []error{err}
-	}
-	lprogram := &loader.Program{
-		Build: &build.Context{
-			GOARCH:      c.GOARCH(),
-			GOOS:        c.GOOS(),
-			GOROOT:      goroot,
-			GOPATH:      goenv.Get("GOPATH"),
-			CgoEnabled:  c.CgoEnabled(),
-			UseAllFiles: false,
-			Compiler:    "gc", // must be one of the recognized compilers
-			BuildTags:   c.BuildTags(),
-		},
-		Tests: c.TestConfig.CompileTestBinary,
-		TypeChecker: types.Config{
 		Sizes: &stdSizes{
 			IntSize:  int64(c.targetData.TypeAllocSize(c.intType)),
 			PtrSize:  int64(c.targetData.PointerSize()),
 			MaxAlign: int64(c.targetData.PrefTypeAlignment(c.i8ptrType)),
-			},
+		}})
-		},
-		Dir:          wd,
-		TINYGOROOT:   goenv.Get("TINYGOROOT"),
-		CFlags:       c.CFlags(),
-		ClangHeaders: c.ClangHeaders,
-	}
-
-	err = lprogram.Load(pkgName)
 	if err != nil {
 		return c.mod, nil, nil, []error{err}
 	}
@@ -361,11 +330,8 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 	// Gather the list of (C) file paths that should be included in the build.
 	var extraFiles []string
 	for _, pkg := range c.ir.LoaderProgram.Sorted() {
-		for _, file := range pkg.OtherFiles {
+		for _, filename := range pkg.CFiles {
-			switch strings.ToLower(filepath.Ext(file)) {
+			extraFiles = append(extraFiles, filepath.Join(pkg.Dir, filename))
-			case ".c":
-				extraFiles = append(extraFiles, file)
-			}
 		}
 	}
 

ir/ir.go

@@ -67,7 +67,7 @@ func NewProgram(lprogram *loader.Program) *Program {
 	program := lprogram.LoadSSA()
 	program.Build()
 
-	mainPkg := program.ImportedPackage(lprogram.MainPkg.PkgPath)
+	mainPkg := program.ImportedPackage(lprogram.MainPkg().ImportPath)
 	if mainPkg == nil {
 		panic("could not find main package")
 	}
@@ -79,7 +79,7 @@ func NewProgram(lprogram *loader.Program) *Program {
 	}
 
 	for _, pkg := range lprogram.Sorted() {
-		p.AddPackage(program.ImportedPackage(pkg.PkgPath))
+		p.AddPackage(program.ImportedPackage(pkg.ImportPath))
 	}
 
 	return p

loader/errors.go

@@ -1,5 +1,7 @@
 package loader
 
+import "go/scanner"
+
 // Errors contains a list of parser errors or a list of typechecker errors for
 // the given package.
 type Errors struct {
@@ -10,3 +12,14 @@ type Errors struct {
 func (e Errors) Error() string {
 	return "could not compile: " + e.Errs[0].Error()
 }
+
+// Error is a regular error but with an added import stack. This is especially
+// useful for debugging import cycle errors.
+type Error struct {
+	ImportStack []string
+	Err         scanner.Error
+}
+
+func (e Error) Error() string {
+	return e.Err.Error()
+}

loader/goroot.go

@@ -212,7 +212,7 @@ func pathsToOverride(needsSyscallPackage bool) map[string]bool {
 		"reflect/":              false,
 		"runtime/":              false,
 		"sync/":                 true,
-		"testing/":              false,
+		"testing/":              true,
 	}
 	if needsSyscallPackage {
 		paths["syscall/"] = true // include syscall/js

loader/list.go

@@ -0,0 +1,30 @@
+package loader
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/tinygo-org/tinygo/compileopts"
+)
+
+// List returns a ready-to-run *exec.Cmd for running the `go list` command with
+// the configuration used for TinyGo.
+func List(config *compileopts.Config, extraArgs, pkgs []string) (*exec.Cmd, error) {
+	goroot, err := GetCachedGoroot(config)
+	if err != nil {
+		return nil, err
+	}
+	args := append([]string{"list"}, extraArgs...)
+	if len(config.BuildTags()) != 0 {
+		args = append(args, "-tags", strings.Join(config.BuildTags(), " "))
+	}
+	args = append(args, pkgs...)
+	cgoEnabled := "0"
+	if config.CgoEnabled() {
+		cgoEnabled = "1"
+	}
+	cmd := exec.Command("go", args...)
+	cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled)
+	return cmd, nil
+}

loader/loader.go

@@ -2,124 +2,147 @@ package loader
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"go/ast"
-	"go/build"
 	"go/parser"
 	"go/scanner"
 	"go/token"
 	"go/types"
+	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
-	"text/template"
+	"syscall"
 
 	"github.com/tinygo-org/tinygo/cgo"
+	"github.com/tinygo-org/tinygo/compileopts"
 	"github.com/tinygo-org/tinygo/goenv"
-	"golang.org/x/tools/go/packages"
 )
 
 // Program holds all packages and some metadata about the program as a whole.
 type Program struct {
-	Build        *build.Context
+	config       *compileopts.Config
-	Tests        bool
+	clangHeaders string
+	typeChecker  types.Config
+	goroot       string // synthetic GOROOT
+	workingDir   string
+
 	Packages map[string]*Package
-	MainPkg      *Package
 	sorted   []*Package
 	fset     *token.FileSet
-	TypeChecker  types.Config
+
-	Dir          string // current working directory (for error reporting)
+	// Information obtained during parsing.
-	TINYGOROOT   string // root of the TinyGo installation or root of the source code
-	CFlags       []string
 	LDFlags []string
-	ClangHeaders string
+}
+
+// PackageJSON is a subset of the JSON struct returned from `go list`.
+type PackageJSON struct {
+	Dir        string
+	ImportPath string
+	ForTest    string
+
+	// Source files
+	GoFiles  []string
+	CgoFiles []string
+	CFiles   []string
+
+	// Dependency information
+	Imports   []string
+	ImportMap map[string]string
+
+	// Error information
+	Error *struct {
+		ImportStack []string
+		Pos         string
+		Err         string
+	}
 }
 
 // Package holds a loaded package, its imports, and its parsed files.
 type Package struct {
-	*Program
+	PackageJSON
-	*packages.Package
+
+	program *Program
 	Files   []*ast.File
 	Pkg     *types.Package
-	types.Info
+	info    types.Info
 }
 
 // Load loads the given package with all dependencies (including the runtime
 // package). Call .Parse() afterwards to parse all Go files (including CGo
 // processing, if necessary).
-func (p *Program) Load(importPath string) error {
+func Load(config *compileopts.Config, inputPkgs []string, clangHeaders string, typeChecker types.Config) (*Program, error) {
-	if p.Packages == nil {
+	goroot, err := GetCachedGoroot(config)
-		p.Packages = make(map[string]*Package)
+	if err != nil {
+		return nil, err
+	}
+	wd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	p := &Program{
+		config:       config,
+		clangHeaders: clangHeaders,
+		typeChecker:  typeChecker,
+		goroot:       goroot,
+		workingDir:   wd,
+		Packages:     make(map[string]*Package),
+		fset:         token.NewFileSet(),
 	}
 
-	err := p.loadPackage(importPath)
+	// List the dependencies of this package, in raw JSON format.
+	extraArgs := []string{"-json", "-deps"}
+	if config.TestConfig.CompileTestBinary {
+		extraArgs = append(extraArgs, "-test")
+	}
+	cmd, err := List(config, extraArgs, inputPkgs)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	p.MainPkg = p.sorted[len(p.sorted)-1]
+	buf := &bytes.Buffer{}
-	if _, ok := p.Packages["runtime"]; !ok {
+	cmd.Stdout = buf
-		// The runtime package wasn't loaded. Although `go list -deps` seems to
+	cmd.Stderr = os.Stderr
-		// return the full dependency list, there is no way to get those
+	err = cmd.Run()
-		// packages from the go/packages package. Therefore load the runtime
+	if err != nil {
-		// manually and add it to the list of to-be-compiled packages
+		if exitErr, ok := err.(*exec.ExitError); ok {
-		// (duplicates are already filtered).
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
-		return p.loadPackage("runtime")
+				os.Exit(status.ExitStatus())
+			}
+			os.Exit(1)
+		}
+		return nil, fmt.Errorf("failed to run `go list`: %s", err)
 	}
-	return nil
-}
 
-func (p *Program) loadPackage(importPath string) error {
+	// Parse the returned json from `go list`.
-	cgoEnabled := "0"
+	decoder := json.NewDecoder(buf)
-	if p.Build.CgoEnabled {
+	for {
-		cgoEnabled = "1"
+		pkg := &Package{
+			program: p,
+			info: types.Info{
+				Types:      make(map[ast.Expr]types.TypeAndValue),
+				Defs:       make(map[*ast.Ident]types.Object),
+				Uses:       make(map[*ast.Ident]types.Object),
+				Implicits:  make(map[ast.Node]types.Object),
+				Scopes:     make(map[ast.Node]*types.Scope),
+				Selections: make(map[*ast.SelectorExpr]*types.Selection),
+			},
 		}
-	pkgs, err := packages.Load(&packages.Config{
+		err := decoder.Decode(&pkg.PackageJSON)
-		Mode:       packages.NeedName | packages.NeedFiles | packages.NeedImports | packages.NeedDeps,
-		Env:        append(os.Environ(), "GOROOT="+p.Build.GOROOT, "GOOS="+p.Build.GOOS, "GOARCH="+p.Build.GOARCH, "CGO_ENABLED="+cgoEnabled),
-		BuildFlags: []string{"-tags", strings.Join(p.Build.BuildTags, " ")},
-		Tests:      p.Tests,
-	}, importPath)
 		if err != nil {
-		return err
+			if err == io.EOF {
+				break
 			}
-	var pkg *packages.Package
+			return nil, err
-	if p.Tests {
-		// We need the second package. Quoting from the docs:
-		// > For example, when using the go command, loading "fmt" with Tests=true
-		// > returns four packages, with IDs "fmt" (the standard package),
-		// > "fmt [fmt.test]" (the package as compiled for the test),
-		// > "fmt_test" (the test functions from source files in package fmt_test),
-		// > and "fmt.test" (the test binary).
-		pkg = pkgs[1]
-	} else {
-		if len(pkgs) != 1 {
-			return fmt.Errorf("expected exactly one package while importing %s, got %d", importPath, len(pkgs))
 		}
-		pkg = pkgs[0]
+		if pkg.Error != nil {
-	}
+			// There was an error while importing (for example, a circular
-	var importError *Errors
+			// dependency).
-	var addPackages func(pkg *packages.Package)
-	addPackages = func(pkg *packages.Package) {
-		if _, ok := p.Packages[pkg.PkgPath]; ok {
-			return
-		}
-		pkg2 := p.newPackage(pkg)
-		p.Packages[pkg.PkgPath] = pkg2
-		if len(pkg.Errors) != 0 {
-			if importError != nil {
-				// There was another error reported already. Do not report
-				// errors from multiple packages at once.
-				return
-			}
-			importError = &Errors{
-				Pkg: pkg2,
-			}
-			for _, err := range pkg.Errors {
 			pos := token.Position{}
-				fields := strings.Split(err.Pos, ":")
+			fields := strings.Split(pkg.Error.Pos, ":")
 			if len(fields) >= 2 {
 				// There is some file/line/column information.
 				if n, err := strconv.Atoi(fields[len(fields)-2]); err == nil {
@@ -134,33 +157,23 @@ func (p *Program) loadPackage(importPath string) error {
 				}
 				pos.Filename = p.getOriginalPath(pos.Filename)
 			}
-				importError.Errs = append(importError.Errs, scanner.Error{
+			err := scanner.Error{
 				Pos: pos,
-					Msg: err.Msg,
+				Msg: pkg.Error.Err,
-				})
 			}
-			return
+			if len(pkg.Error.ImportStack) != 0 {
+				return nil, Error{
+					ImportStack: pkg.Error.ImportStack,
+					Err:         err,
+				}
+			}
+			return nil, err
+		}
+		p.sorted = append(p.sorted, pkg)
+		p.Packages[pkg.ImportPath] = pkg
 	}
 
-		// Get the list of imports (sorted alphabetically).
+	return p, nil
-		names := make([]string, 0, len(pkg.Imports))
-		for name := range pkg.Imports {
-			names = append(names, name)
-		}
-		sort.Strings(names)
-
-		// Add all the imports.
-		for _, name := range names {
-			addPackages(pkg.Imports[name])
-		}
-
-		p.sorted = append(p.sorted, pkg2)
-	}
-	addPackages(pkg)
-	if importError != nil {
-		return *importError
-	}
-	return nil
 }
 
 // getOriginalPath looks whether this path is in the generated GOROOT and if so,
@@ -168,23 +181,23 @@ func (p *Program) loadPackage(importPath string) error {
 // the input path is returned.
 func (p *Program) getOriginalPath(path string) string {
 	originalPath := path
-	if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) {
+	if strings.HasPrefix(path, p.goroot+string(filepath.Separator)) {
 		// If this file is part of the synthetic GOROOT, try to infer the
 		// original path.
-		relpath := path[len(filepath.Join(p.Build.GOROOT, "src"))+1:]
+		relpath := path[len(filepath.Join(p.goroot, "src"))+1:]
 		realgorootPath := filepath.Join(goenv.Get("GOROOT"), "src", relpath)
 		if _, err := os.Stat(realgorootPath); err == nil {
 			originalPath = realgorootPath
 		}
 		maybeInTinyGoRoot := false
-		for prefix := range pathsToOverride(needsSyscallPackage(p.Build.BuildTags)) {
+		for prefix := range pathsToOverride(needsSyscallPackage(p.config.BuildTags())) {
 			if !strings.HasPrefix(relpath, prefix) {
 				continue
 			}
 			maybeInTinyGoRoot = true
 		}
 		if maybeInTinyGoRoot {
-			tinygoPath := filepath.Join(p.TINYGOROOT, "src", relpath)
+			tinygoPath := filepath.Join(goenv.Get("TINYGOROOT"), "src", relpath)
 			if _, err := os.Stat(tinygoPath); err == nil {
 				originalPath = tinygoPath
 			}
@@ -193,28 +206,18 @@ func (p *Program) getOriginalPath(path string) string {
 	return originalPath
 }
 
-// newPackage instantiates a new *Package object with initialized members.
-func (p *Program) newPackage(pkg *packages.Package) *Package {
-	return &Package{
-		Program: p,
-		Package: pkg,
-		Info: types.Info{
-			Types:      make(map[ast.Expr]types.TypeAndValue),
-			Defs:       make(map[*ast.Ident]types.Object),
-			Uses:       make(map[*ast.Ident]types.Object),
-			Implicits:  make(map[ast.Node]types.Object),
-			Scopes:     make(map[ast.Node]*types.Scope),
-			Selections: make(map[*ast.SelectorExpr]*types.Selection),
-		},
-	}
-}
-
 // Sorted returns a list of all packages, sorted in a way that no packages come
 // before the packages they depend upon.
 func (p *Program) Sorted() []*Package {
 	return p.sorted
 }
 
+// MainPkg returns the last package in the Sorted() slice. This is the main
+// package of the program.
+func (p *Program) MainPkg() *Package {
+	return p.sorted[len(p.sorted)-1]
+}
+
 // Parse parses all packages and typechecks them.
 //
 // The returned error may be an Errors error, which contains a list of errors.
@@ -222,22 +225,16 @@ func (p *Program) Sorted() []*Package {
 // Idempotent.
 func (p *Program) Parse() error {
 	// Parse all packages.
-	for _, pkg := range p.Sorted() {
+	// TODO: do this in parallel.
+	for _, pkg := range p.sorted {
 		err := pkg.Parse()
 		if err != nil {
 			return err
 		}
 	}
 
-	if p.Tests {
-		err := p.swapTestMain()
-		if err != nil {
-			return err
-		}
-	}
-
 	// Typecheck all packages.
-	for _, pkg := range p.Sorted() {
+	for _, pkg := range p.sorted {
 		err := pkg.Check()
 		if err != nil {
 			return err
@@ -247,82 +244,6 @@ func (p *Program) Parse() error {
 	return nil
 }
 
-func (p *Program) swapTestMain() error {
-	var tests []string
-
-	isTestFunc := func(f *ast.FuncDecl) bool {
-		// TODO: improve signature check
-		if strings.HasPrefix(f.Name.Name, "Test") && f.Name.Name != "TestMain" {
-			return true
-		}
-		return false
-	}
-	for _, f := range p.MainPkg.Files {
-		for i, d := range f.Decls {
-			switch v := d.(type) {
-			case *ast.FuncDecl:
-				if isTestFunc(v) {
-					tests = append(tests, v.Name.Name)
-				}
-				if v.Name.Name == "main" {
-					// Remove main
-					if len(f.Decls) == 1 {
-						f.Decls = make([]ast.Decl, 0)
-					} else {
-						f.Decls[i] = f.Decls[len(f.Decls)-1]
-						f.Decls = f.Decls[:len(f.Decls)-1]
-					}
-				}
-			}
-		}
-	}
-
-	// TODO: Check if they defined a TestMain and call it instead of testing.TestMain
-	const mainBody = `package main
-
-import (
-	"testing"
-)
-
-func main () {
-	m := &testing.M{
-		Tests: []testing.TestToCall{
-{{range .TestFunctions}}
-			{Name: "{{.}}", Func: {{.}}},
-{{end}}
-		},
-	}
-
-	testing.TestMain(m)
-}
-`
-	tmpl := template.Must(template.New("testmain").Parse(mainBody))
-	b := bytes.Buffer{}
-	tmplData := struct {
-		TestFunctions []string
-	}{
-		TestFunctions: tests,
-	}
-
-	err := tmpl.Execute(&b, tmplData)
-	if err != nil {
-		return err
-	}
-	path := filepath.Join(p.MainPkg.Dir, "$testmain.go")
-
-	if p.fset == nil {
-		p.fset = token.NewFileSet()
-	}
-
-	newMain, err := parser.ParseFile(p.fset, path, b.Bytes(), parser.AllErrors)
-	if err != nil {
-		return err
-	}
-	p.MainPkg.Files = append(p.MainPkg.Files, newMain)
-
-	return nil
-}
-
 // parseFile is a wrapper around parser.ParseFile.
 func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
 	if p.fset == nil {
@@ -342,14 +263,13 @@ func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
 // Idempotent.
 func (p *Package) Parse() error {
 	if len(p.Files) != 0 {
-		return nil
+		return nil // nothing to do (?)
 	}
 
 	// Load the AST.
-	// TODO: do this in parallel.
+	if p.ImportPath == "unsafe" {
-	if p.PkgPath == "unsafe" {
+		// Special case for the unsafe package, which is defined internally by
-		// Special case for the unsafe package. Don't even bother loading
+		// the types package.
-		// the files.
 		p.Pkg = types.Unsafe
 		return nil
 	}
@@ -369,11 +289,11 @@ func (p *Package) Parse() error {
 // Idempotent.
 func (p *Package) Check() error {
 	if p.Pkg != nil {
-		return nil
+		return nil // already typechecked
 	}
 
 	var typeErrors []error
-	checker := p.TypeChecker
+	checker := p.program.typeChecker // make a copy, because it will be modified
 	checker.Error = func(err error) {
 		typeErrors = append(typeErrors, err)
 	}
@@ -381,7 +301,7 @@ func (p *Package) Check() error {
 	// Do typechecking of the package.
 	checker.Importer = p
 
-	typesPkg, err := checker.Check(p.PkgPath, p.fset, p.Files, &p.Info)
+	typesPkg, err := checker.Check(p.ImportPath, p.program.fset, p.Files, &p.info)
 	if err != nil {
 		if err, ok := err.(Errors); ok {
 			return err
@@ -394,40 +314,46 @@ func (p *Package) Check() error {
 
 // parseFiles parses the loaded list of files and returns this list.
 func (p *Package) parseFiles() ([]*ast.File, error) {
-	// TODO: do this concurrently.
 	var files []*ast.File
 	var fileErrs []error
 
-	var cgoFiles []*ast.File
+	// Parse all files (incuding CgoFiles).
-	for _, file := range p.GoFiles {
+	parseFile := func(file string) {
-		f, err := p.parseFile(file, parser.ParseComments)
+		if !filepath.IsAbs(file) {
+			file = filepath.Join(p.Dir, file)
+		}
+		f, err := p.program.parseFile(file, parser.ParseComments)
 		if err != nil {
 			fileErrs = append(fileErrs, err)
-			continue
+			return
-		}
-		if err != nil {
-			fileErrs = append(fileErrs, err)
-			continue
-		}
-		for _, importSpec := range f.Imports {
-			if importSpec.Path.Value == `"C"` {
-				cgoFiles = append(cgoFiles, f)
-			}
 		}
 		files = append(files, f)
 	}
-	if len(cgoFiles) != 0 {
+	for _, file := range p.GoFiles {
-		cflags := append(p.CFlags, "-I"+filepath.Dir(p.GoFiles[0]))
+		parseFile(file)
-		if p.ClangHeaders != "" {
-			cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.ClangHeaders)
 	}
-		generated, ldflags, errs := cgo.Process(files, p.Program.Dir, p.fset, cflags)
+	for _, file := range p.CgoFiles {
+		parseFile(file)
+	}
+
+	// Do CGo processing.
+	if len(p.CgoFiles) != 0 {
+		var cflags []string
+		cflags = append(cflags, p.program.config.CFlags()...)
+		cflags = append(cflags, "-I"+p.Dir)
+		if p.program.clangHeaders != "" {
+			cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.program.clangHeaders)
+		}
+		generated, ldflags, errs := cgo.Process(files, p.program.workingDir, p.program.fset, cflags)
 		if errs != nil {
 			fileErrs = append(fileErrs, errs...)
 		}
 		files = append(files, generated)
-		p.LDFlags = append(p.LDFlags, ldflags...)
+		p.program.LDFlags = append(p.program.LDFlags, ldflags...)
 	}
+
+	// Only return an error after CGo processing, so that errors in parsing and
+	// CGo can be reported together.
 	if len(fileErrs) != 0 {
 		return nil, Errors{p, fileErrs}
 	}
@@ -441,8 +367,13 @@ func (p *Package) Import(to string) (*types.Package, error) {
 	if to == "unsafe" {
 		return types.Unsafe, nil
 	}
-	if _, ok := p.Imports[to]; ok {
+	if replace, ok := p.ImportMap[to]; ok {
-		return p.Packages[p.Imports[to].PkgPath].Pkg, nil
+		// This import path should be replaced by another import path, according
+		// to `go list`.
+		to = replace
+	}
+	if imported, ok := p.program.Packages[to]; ok {
+		return imported.Pkg, nil
 	} else {
 		return nil, errors.New("package not imported: " + to)
 	}

loader/ssa.go

@@ -10,8 +10,8 @@ import (
 func (p *Program) LoadSSA() *ssa.Program {
 	prog := ssa.NewProgram(p.fset, ssa.SanityCheckFunctions|ssa.BareInits|ssa.GlobalDebug)
 
-	for _, pkg := range p.Sorted() {
+	for _, pkg := range p.sorted {
-		prog.CreatePackage(pkg.Pkg, pkg.Files, &pkg.Info, true)
+		prog.CreatePackage(pkg.Pkg, pkg.Files, &pkg.info, true)
 	}
 
 	return prog

main.go

@@ -126,12 +126,6 @@ func Test(pkgName string, options *compileopts.Options) error {
 		return err
 	}
 
-	// Add test build tag. This is incorrect: `go test` only looks at the
-	// _test.go file suffix but does not add the test build tag in the process.
-	// However, it's a simple fix right now.
-	// For details: https://github.com/golang/go/issues/21360
-	config.Target.BuildTags = append(config.Target.BuildTags, "test")
-
 	return builder.Build(pkgName, ".elf", config, func(tmppath string) error {
 		cmd := exec.Command(tmppath)
 		cmd.Stdout = os.Stdout
@@ -648,36 +642,6 @@ func getDefaultPort() (port string, err error) {
 	return d[0], nil
 }
 
-// runGoList runs the `go list` command but using the configuration used for
-// TinyGo.
-func runGoList(config *compileopts.Config, flagJSON, flagDeps bool, pkgs []string) error {
-	goroot, err := loader.GetCachedGoroot(config)
-	if err != nil {
-		return err
-	}
-	args := []string{"list"}
-	if flagJSON {
-		args = append(args, "-json")
-	}
-	if flagDeps {
-		args = append(args, "-deps")
-	}
-	if len(config.BuildTags()) != 0 {
-		args = append(args, "-tags", strings.Join(config.BuildTags(), " "))
-	}
-	args = append(args, pkgs...)
-	cgoEnabled := "0"
-	if config.CgoEnabled() {
-		cgoEnabled = "1"
-	}
-	cmd := exec.Command("go", args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled)
-	cmd.Run()
-	return nil
-}
-
 func usage() {
 	fmt.Fprintln(os.Stderr, "TinyGo is a Go compiler for small places.")
 	fmt.Fprintln(os.Stderr, "version:", goenv.Version)
@@ -751,10 +715,16 @@ func printCompilerError(logln func(...interface{}), err error) {
 			}
 		}
 	case loader.Errors:
-		logln("#", err.Pkg.PkgPath)
+		logln("#", err.Pkg.ImportPath)
 		for _, err := range err.Errs {
 			printCompilerError(logln, err)
 		}
+	case loader.Error:
+		logln(err.Err.Error())
+		logln("package", err.ImportStack[0])
+		for _, pkgPath := range err.ImportStack[1:] {
+			logln("\timports", pkgPath)
+		}
 	case *builder.MultiError:
 		for _, err := range err.Errs {
 			printCompilerError(logln, err)
@@ -982,11 +952,31 @@ func main() {
 			usage()
 			os.Exit(1)
 		}
-		err = runGoList(config, *flagJSON, *flagDeps, flag.Args())
+		var extraArgs []string
+		if *flagJSON {
+			extraArgs = append(extraArgs, "-json")
+		}
+		if *flagDeps {
+			extraArgs = append(extraArgs, "-deps")
+		}
+		cmd, err := loader.List(config, extraArgs, flag.Args())
 		if err != nil {
 			fmt.Fprintln(os.Stderr, "failed to run `go list`:", err)
 			os.Exit(1)
 		}
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			if exitErr, ok := err.(*exec.ExitError); ok {
+				if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+					os.Exit(status.ExitStatus())
+				}
+				os.Exit(1)
+			}
+			fmt.Fprintln(os.Stderr, "failed to run `go list`:", err)
+			os.Exit(1)
+		}
 	case "clean":
 		// remove cache directory
 		err := os.RemoveAll(goenv.Get("GOCACHE"))

src/runtime/pprof/pprof.go

@@ -0,0 +1,29 @@
+package pprof
+
+// TinyGo does not implement pprof. However, a dummy shell is needed for the
+// testing package (and testing/internal/pprof).
+
+import (
+	"errors"
+	"io"
+)
+
+var ErrUnimplemented = errors.New("runtime/pprof: unimplemented")
+
+type Profile struct {
+}
+
+func StartCPUProfile(w io.Writer) error {
+	return nil
+}
+
+func StopCPUProfile() {
+}
+
+func Lookup(name string) *Profile {
+	return nil
+}
+
+func (p *Profile) WriteTo(w io.Writer, debug int) error {
+	return ErrUnimplemented
+}

src/testing/benchmark.go

@@ -15,3 +15,8 @@ type B struct {
 	common
 	N int
 }
+
+type InternalBenchmark struct {
+	Name string
+	F    func(b *B)
+}

src/testing/testing.go

@@ -154,18 +154,16 @@ func (c *common) Skipped() bool {
 	return c.skipped
 }
 
-// TestToCall is a reference to a test that should be called during a test suite run.
+// InternalTest is a reference to a test that should be called during a test suite run.
-type TestToCall struct {
+type InternalTest struct {
-	// Name of the test to call.
 	Name string
-	// Function reference to the test.
+	F    func(*T)
-	Func func(*T)
 }
 
 // M is a test suite.
 type M struct {
 	// tests is a list of the test names to execute
-	Tests []TestToCall
+	Tests []InternalTest
 }
 
 // Run the test suite.
@@ -180,7 +178,7 @@ func (m *M) Run() int {
 		}
 
 		fmt.Printf("=== RUN   %s\n", test.Name)
-		test.Func(t)
+		test.F(t)
 
 		if t.failed {
 			fmt.Printf("--- FAIL: %s\n", test.Name)
@@ -204,3 +202,16 @@ func (m *M) Run() int {
 func TestMain(m *M) {
 	os.Exit(m.Run())
 }
+
+func MainStart(deps interface{}, tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) *M {
+	return &M{
+		Tests: tests,
+	}
+}
+
+type InternalExample struct {
+	Name      string
+	F         func()
+	Output    string
+	Unordered bool
+}