loader: merge roots from both Go and TinyGo in a cached directory

This commit changes the way that packages are looked up. Instead of
working around the loader package by modifying the GOROOT variable for
specific packages, create a new GOROOT using symlinks. This GOROOT is
cached for the specified configuration (Go version, underlying GOROOT
path, TinyGo path, whether to override the syscall package).

This will also enable go module support in the future.

Windows is a bit harder to support, because it only allows the creation
of symlinks when developer mode is enabled. This is worked around by
using symlinks and if that fails, using directory junctions or hardlinks
instead. This should work in the vast majority of cases. The only case
it doesn't work, is if developer mode is disabled and TinyGo, the Go
toolchain, and the cache directory are not all on the same filesystem.
If this is a problem, it is still possible to improve the code by using
file copies instead.

As a side effect, this also makes diagnostics use a relative file path
only when the file is not in GOROOT or in TINYGOROOT.

compiler/compiler.go

@@ -137,64 +137,25 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 	c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace()
 	dummyFunc.EraseFromParentAsFunction()
 
-	// Prefix the GOPATH with the system GOROOT, as GOROOT is already set to
-	// the TinyGo root.
-	overlayGopath := goenv.Get("GOPATH")
-	if overlayGopath == "" {
-		overlayGopath = goenv.Get("GOROOT")
-	} else {
-		overlayGopath = goenv.Get("GOROOT") + string(filepath.ListSeparator) + overlayGopath
-	}
-
 	wd, err := os.Getwd()
 	if err != nil {
 		return c.mod, nil, nil, []error{err}
 	}
+	goroot, err := loader.GetCachedGoroot(c.Config)
+	if err != nil {
+		return c.mod, nil, []error{err}
+	}
 	lprogram := &loader.Program{
 		Build: &build.Context{
 			GOARCH:      c.GOARCH(),
 			GOOS:        c.GOOS(),
-			GOROOT:      goenv.Get("GOROOT"),
+			GOROOT:      goroot,
 			GOPATH:      goenv.Get("GOPATH"),
 			CgoEnabled:  c.CgoEnabled(),
 			UseAllFiles: false,
 			Compiler:    "gc", // must be one of the recognized compilers
 			BuildTags:   c.BuildTags(),
 		},
-		OverlayBuild: &build.Context{
-			GOARCH:      c.GOARCH(),
-			GOOS:        c.GOOS(),
-			GOROOT:      goenv.Get("TINYGOROOT"),
-			GOPATH:      overlayGopath,
-			CgoEnabled:  c.CgoEnabled(),
-			UseAllFiles: false,
-			Compiler:    "gc", // must be one of the recognized compilers
-			BuildTags:   c.BuildTags(),
-		},
-		OverlayPath: func(path string) string {
-			// Return the (overlay) import path when it should be overlaid, and
-			// "" if it should not.
-			if strings.HasPrefix(path, tinygoPath+"/src/") {
-				// Avoid issues with packages that are imported twice, one from
-				// GOPATH and one from TINYGOPATH.
-				path = path[len(tinygoPath+"/src/"):]
-			}
-			switch path {
-			case "machine", "os", "reflect", "runtime", "runtime/interrupt", "runtime/volatile", "sync", "testing", "internal/reflectlite", "internal/bytealg", "internal/task":
-				return path
-			default:
-				if strings.HasPrefix(path, "device/") || strings.HasPrefix(path, "examples/") {
-					return path
-				} else if path == "syscall" {
-					for _, tag := range c.BuildTags() {
-						if tag == "baremetal" || tag == "darwin" {
-							return path
-						}
-					}
-				}
-			}
-			return ""
-		},
 		TypeChecker: types.Config{
 			Sizes: &stdSizes{
 				IntSize:  int64(c.targetData.TypeAllocSize(c.intType)),

loader/goroot.go

@@ -0,0 +1,240 @@
+package loader
+
+// This file constructs a new temporary GOROOT directory by merging both the
+// standard Go GOROOT and the GOROOT from TinyGo using symlinks.
+
+import (
+	"crypto/sha512"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strconv"
+
+	"github.com/tinygo-org/tinygo/compileopts"
+	"github.com/tinygo-org/tinygo/goenv"
+)
+
+// GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and
+// the GOROOT from TinyGo using lots of symbolic links.
+func GetCachedGoroot(config *compileopts.Config) (string, error) {
+	goroot := goenv.Get("GOROOT")
+	if goroot == "" {
+		return "", errors.New("could not determine GOROOT")
+	}
+	tinygoroot := goenv.Get("TINYGOROOT")
+	if tinygoroot == "" {
+		return "", errors.New("could not determine TINYGOROOT")
+	}
+
+	// Determine the location of the cached GOROOT.
+	version, err := goenv.GorootVersionString(goroot)
+	if err != nil {
+		return "", err
+	}
+	// This hash is really a cache key, that contains (hopefully) enough
+	// information to make collisions unlikely during development.
+	// By including the Go version and TinyGo version, cache collisions should
+	// not happen outside of development.
+	hash := sha512.New512_256()
+	fmt.Fprintln(hash, goroot)
+	fmt.Fprintln(hash, version)
+	fmt.Fprintln(hash, goenv.Version)
+	fmt.Fprintln(hash, tinygoroot)
+	gorootsHash := hash.Sum(nil)
+	gorootsHashHex := hex.EncodeToString(gorootsHash[:])
+	cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), "goroot-"+version+"-"+gorootsHashHex)
+	if needsSyscallPackage(config.BuildTags()) {
+		cachedgoroot += "-syscall"
+	}
+
+	if _, err := os.Stat(cachedgoroot); err == nil {
+		return cachedgoroot, nil
+	}
+	tmpgoroot := cachedgoroot + ".tmp" + strconv.Itoa(rand.Int())
+	err = os.MkdirAll(tmpgoroot, 0777)
+	if err != nil {
+		return "", err
+	}
+
+	// Remove the temporary directory if it wasn't moved to the right place
+	// (for example, when there was an error).
+	defer os.RemoveAll(tmpgoroot)
+
+	for _, name := range []string{"bin", "lib", "pkg"} {
+		err = symlink(filepath.Join(goroot, name), filepath.Join(tmpgoroot, name))
+		if err != nil {
+			return "", err
+		}
+	}
+	err = mergeDirectory(goroot, tinygoroot, tmpgoroot, "", pathsToOverride(needsSyscallPackage(config.BuildTags())))
+	if err != nil {
+		return "", err
+	}
+	err = os.Rename(tmpgoroot, cachedgoroot)
+	if err != nil {
+		if os.IsExist(err) {
+			// Another invocation of TinyGo also seems to have created a GOROOT.
+			// Use that one instead. Our new GOROOT will be automatically
+			// deleted by the defer above.
+			return cachedgoroot, nil
+		}
+		return "", err
+	}
+	return cachedgoroot, nil
+}
+
+// mergeDirectory merges two roots recursively. The tmpgoroot is the directory
+// that will be created by this call by either symlinking the directory from
+// goroot or tinygoroot, or by creating the directory and merging the contents.
+func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides map[string]bool) error {
+	if mergeSubdirs, ok := overrides[importPath+"/"]; ok {
+		if !mergeSubdirs {
+			// This directory and all subdirectories should come from the TinyGo
+			// root, so simply make a symlink.
+			newname := filepath.Join(tmpgoroot, "src", importPath)
+			oldname := filepath.Join(tinygoroot, "src", importPath)
+			return symlink(oldname, newname)
+		}
+
+		// Merge subdirectories. Start by making the directory to merge.
+		err := os.Mkdir(filepath.Join(tmpgoroot, "src", importPath), 0777)
+		if err != nil {
+			return err
+		}
+
+		// Symlink all files from TinyGo, and symlink directories from TinyGo
+		// that need to be overridden.
+		tinygoEntries, err := ioutil.ReadDir(filepath.Join(tinygoroot, "src", importPath))
+		if err != nil {
+			return err
+		}
+		for _, e := range tinygoEntries {
+			if e.IsDir() {
+				// A directory, so merge this thing.
+				err := mergeDirectory(goroot, tinygoroot, tmpgoroot, path.Join(importPath, e.Name()), overrides)
+				if err != nil {
+					return err
+				}
+			} else {
+				// A file, so symlink this.
+				newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+				oldname := filepath.Join(tinygoroot, "src", importPath, e.Name())
+				err := symlink(oldname, newname)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// Symlink all directories from $GOROOT that are not part of the TinyGo
+		// overrides.
+		gorootEntries, err := ioutil.ReadDir(filepath.Join(goroot, "src", importPath))
+		if err != nil {
+			return err
+		}
+		for _, e := range gorootEntries {
+			if !e.IsDir() {
+				// Don't merge in files from Go. Otherwise we'd end up with a
+				// weird syscall package with files from both roots.
+				continue
+			}
+			if _, ok := overrides[path.Join(importPath, e.Name())+"/"]; ok {
+				// Already included above, so don't bother trying to create this
+				// symlink.
+				continue
+			}
+			newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+			oldname := filepath.Join(goroot, "src", importPath, e.Name())
+			err := symlink(oldname, newname)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// needsSyscallPackage returns whether the syscall package should be overriden
+// with the TinyGo version. This is the case on some targets.
+func needsSyscallPackage(buildTags []string) bool {
+	for _, tag := range buildTags {
+		if tag == "baremetal" || tag == "darwin" {
+			return true
+		}
+	}
+	return false
+}
+
+// The boolean indicates whether to merge the subdirs. True means merge, false
+// means use the TinyGo version.
+func pathsToOverride(needsSyscallPackage bool) map[string]bool {
+	paths := map[string]bool{
+		"/":                     true,
+		"device/":               false,
+		"examples/":             false,
+		"internal/":             true,
+		"internal/bytealg/":     false,
+		"internal/reflectlite/": false,
+		"internal/task/":        false,
+		"machine/":              false,
+		"os/":                   true,
+		"reflect/":              false,
+		"runtime/":              false,
+		"sync/":                 true,
+		"testing/":              false,
+	}
+	if needsSyscallPackage {
+		paths["syscall/"] = true // include syscall/js
+	}
+	return paths
+}
+
+// symlink creates a symlink or something similar. On Unix-like systems, it
+// always creates a symlink. On Windows, it tries to create a symlink and if
+// that fails, creates a hardlink or directory junction instead.
+//
+// Note that while Windows 10 does support symlinks and allows them to be
+// created using os.Symlink, it requires developer mode to be enabled.
+// Therefore provide a fallback for when symlinking is not possible.
+// Unfortunately this fallback only works when TinyGo is installed on the same
+// filesystem as the TinyGo cache and the Go installation (which is usually the
+// C drive).
+func symlink(oldname, newname string) error {
+	symlinkErr := os.Symlink(oldname, newname)
+	if runtime.GOOS == "windows" && symlinkErr != nil {
+		// Fallback for when developer mode is disabled.
+		// Note that we return the symlink error even if something else fails
+		// later on. This is because symlinks are the easiest to support
+		// (they're also used on Linux and MacOS) and enabling them is easy:
+		// just enable developer mode.
+		st, err := os.Stat(oldname)
+		if err != nil {
+			return symlinkErr
+		}
+		if st.IsDir() {
+			// Make a directory junction. There may be a way to do this
+			// programmatically, but it involves a lot of magic. Use the mklink
+			// command built into cmd instead (mklink is a builtin, not an
+			// external command).
+			err := exec.Command("cmd", "/k", "mklink", "/J", newname, oldname).Run()
+			if err != nil {
+				return symlinkErr
+			}
+		} else {
+			// Make a hard link.
+			err := os.Link(oldname, newname)
+			if err != nil {
+				return symlinkErr
+			}
+		}
+		return nil // success
+	}
+	return symlinkErr
+}

loader/loader.go

@@ -16,14 +16,13 @@ import (
 	"text/template"
 
 	"github.com/tinygo-org/tinygo/cgo"
+	"github.com/tinygo-org/tinygo/goenv"
 )
 
 // Program holds all packages and some metadata about the program as a whole.
 type Program struct {
 	mainPkg      string
 	Build        *build.Context
-	OverlayBuild *build.Context
-	OverlayPath  func(path string) string
 	Packages     map[string]*Package
 	sorted       []*Package
 	fset         *token.FileSet
@@ -55,10 +54,6 @@ func (p *Program) Import(path, srcDir string, pos token.Position) (*Package, err
 
 	// Load this package.
 	ctx := p.Build
-	if newPath := p.OverlayPath(path); newPath != "" {
-		ctx = p.OverlayBuild
-		path = newPath
-	}
 	buildPkg, err := ctx.Import(path, srcDir, build.ImportComment)
 	if err != nil {
 		return nil, scanner.Error{
@@ -320,14 +315,30 @@ func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
 		return nil, err
 	}
 	defer rd.Close()
-	relpath := path
-	if filepath.IsAbs(path) {
-		rp, err := filepath.Rel(p.Dir, path)
-		if err == nil {
-			relpath = rp
+	diagnosticPath := path
+	if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) {
+		// If this file is part of the synthetic GOROOT, try to infer the
+		// original path.
+		relpath := path[len(filepath.Join(p.Build.GOROOT, "src"))+1:]
+		realgorootPath := filepath.Join(goenv.Get("GOROOT"), "src", relpath)
+		if _, err := os.Stat(realgorootPath); err == nil {
+			diagnosticPath = realgorootPath
+		}
+		maybeInTinyGoRoot := false
+		for prefix := range pathsToOverride(needsSyscallPackage(p.Build.BuildTags)) {
+			if !strings.HasPrefix(relpath, prefix) {
+				continue
+			}
+			maybeInTinyGoRoot = true
+		}
+		if maybeInTinyGoRoot {
+			tinygoPath := filepath.Join(p.TINYGOROOT, "src", relpath)
+			if _, err := os.Stat(tinygoPath); err == nil {
+				diagnosticPath = tinygoPath
+			}
 		}
 	}
-	return parser.ParseFile(p.fset, relpath, rd, mode)
+	return parser.ParseFile(p.fset, diagnosticPath, rd, mode)
 }
 
 // Parse parses and typechecks this package.

main.go

@@ -630,6 +630,36 @@ func getDefaultPort() (port string, err error) {
 	return d[0], nil
 }
 
+// runGoList runs the `go list` command but using the configuration used for
+// TinyGo.
+func runGoList(config *compileopts.Config, flagJSON, flagDeps bool, pkgs []string) error {
+	goroot, err := loader.GetCachedGoroot(config)
+	if err != nil {
+		return err
+	}
+	args := []string{"list"}
+	if flagJSON {
+		args = append(args, "-json")
+	}
+	if flagDeps {
+		args = append(args, "-deps")
+	}
+	if len(config.BuildTags()) != 0 {
+		args = append(args, "-tags", strings.Join(config.BuildTags(), " "))
+	}
+	args = append(args, pkgs...)
+	cgoEnabled := "0"
+	if config.CgoEnabled() {
+		cgoEnabled = "1"
+	}
+	cmd := exec.Command("go", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled)
+	cmd.Run()
+	return nil
+}
+
 func usage() {
 	fmt.Fprintln(os.Stderr, "TinyGo is a Go compiler for small places.")
 	fmt.Fprintln(os.Stderr, "version:", goenv.Version)
@@ -641,12 +671,27 @@ func usage() {
 	fmt.Fprintln(os.Stderr, "  flash: compile and flash to the device")
 	fmt.Fprintln(os.Stderr, "  gdb:   run/flash and immediately enter GDB")
 	fmt.Fprintln(os.Stderr, "  env:   list environment variables used during build")
+	fmt.Fprintln(os.Stderr, "  list:  run go list using the TinyGo root")
 	fmt.Fprintln(os.Stderr, "  clean: empty cache directory ("+goenv.Get("GOCACHE")+")")
 	fmt.Fprintln(os.Stderr, "  help:  print this help text")
 	fmt.Fprintln(os.Stderr, "\nflags:")
 	flag.PrintDefaults()
 }
 
+// try to make the path relative to the current working directory. If any error
+// occurs, this error is ignored and the absolute path is returned instead.
+func tryToMakePathRelative(dir string) string {
+	wd, err := os.Getwd()
+	if err != nil {
+		return dir
+	}
+	relpath, err := filepath.Rel(wd, dir)
+	if err != nil {
+		return dir
+	}
+	return relpath
+}
+
 // printCompilerError prints compiler errors using the provided logger function
 // (similar to fmt.Println).
 //
@@ -654,8 +699,24 @@ func usage() {
 // to limitations in the LLVM bindings.
 func printCompilerError(logln func(...interface{}), err error) {
 	switch err := err.(type) {
-	case types.Error, scanner.Error:
+	case types.Error:
+		printCompilerError(logln, scanner.Error{
+			Pos: err.Fset.Position(err.Pos),
+			Msg: err.Msg,
+		})
+	case scanner.Error:
+		if !strings.HasPrefix(err.Pos.Filename, filepath.Join(goenv.Get("GOROOT"), "src")) && !strings.HasPrefix(err.Pos.Filename, filepath.Join(goenv.Get("TINYGOROOT"), "src")) {
+			// This file is not from the standard library (either the GOROOT or
+			// the TINYGOROOT). Make the path relative, for easier reading.
+			// Ignore any errors in the process (falling back to the absolute
+			// path).
+			err.Pos.Filename = tryToMakePathRelative(err.Pos.Filename)
+		}
 		logln(err)
+	case scanner.ErrorList:
+		for _, scannerErr := range err {
+			printCompilerError(logln, *scannerErr)
+		}
 	case *interp.Error:
 		logln("#", err.ImportPath)
 		logln(err.Error())
@@ -674,11 +735,11 @@ func printCompilerError(logln func(...interface{}), err error) {
 	case loader.Errors:
 		logln("#", err.Pkg.ImportPath)
 		for _, err := range err.Errs {
-			logln(err)
+			printCompilerError(logln, err)
 		}
 	case *builder.MultiError:
 		for _, err := range err.Errs {
-			logln(err)
+			printCompilerError(logln, err)
 		}
 	default:
 		logln("error:", err)
@@ -695,6 +756,13 @@ func handleCompilerError(err error) {
 }
 
 func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
+		usage()
+		os.Exit(1)
+	}
+	command := os.Args[1]
+
 	outpath := flag.String("o", "", "output filename")
 	opt := flag.String("opt", "z", "optimization level: 0, 1, 2, s, z")
 	gc := flag.String("gc", "", "garbage collector to use (none, leaking, extalloc, conservative)")
@@ -715,12 +783,11 @@ func main() {
 	wasmAbi := flag.String("wasm-abi", "js", "WebAssembly ABI conventions: js (no i64 params) or generic")
 	heapSize := flag.String("heap-size", "1M", "default heap size in bytes (only supported by WebAssembly)")
 
-	if len(os.Args) < 2 {
-		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
-		usage()
-		os.Exit(1)
+	var flagJSON, flagDeps *bool
+	if command == "list" {
+		flagJSON = flag.Bool("json", false, "print data in JSON format")
+		flagDeps = flag.Bool("deps", false, "")
 	}
-	command := os.Args[1]
 
 	// Early command processing, before commands are interpreted by the Go flag
 	// library.
@@ -886,6 +953,18 @@ func main() {
 		fmt.Printf("build tags:        %s\n", strings.Join(config.BuildTags(), " "))
 		fmt.Printf("garbage collector: %s\n", config.GC())
 		fmt.Printf("scheduler:         %s\n", config.Scheduler())
+	case "list":
+		config, err := builder.NewConfig(options)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			usage()
+			os.Exit(1)
+		}
+		err = runGoList(config, *flagJSON, *flagDeps, flag.Args())
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "failed to run `go list`:", err)
+			os.Exit(1)
+		}
 	case "clean":
 		// remove cache directory
 		err := os.RemoveAll(goenv.Get("GOCACHE"))