loader: switch to custom program loader

Makefile

@@ -70,7 +70,7 @@ clean:
 	@rm -rf build
 
 fmt:
-	@go fmt . ./compiler ./interp ./ir ./src/device/arm ./src/examples/* ./src/machine ./src/runtime ./src/sync
+	@go fmt . ./compiler ./interp ./loader ./ir ./src/device/arm ./src/examples/* ./src/machine ./src/runtime ./src/sync
 	@go fmt ./testdata/*.go
 
 test:

compiler/compiler.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"go/build"
 	"go/constant"
-	"go/parser"
 	"go/token"
 	"go/types"
 	"os"
@@ -17,7 +16,7 @@ import (
 
 	"github.com/aykevl/go-llvm"
 	"github.com/aykevl/tinygo/ir"
-	"golang.org/x/tools/go/loader"
+	"github.com/aykevl/tinygo/loader"
 	"golang.org/x/tools/go/ssa"
 )
 
@@ -183,14 +182,11 @@ func (c *Compiler) Compile(mainPath string) error {
 		gopath = runtime.GOROOT() + string(filepath.ListSeparator) + gopath
 	}
 
-	config := loader.Config{
-		TypeChecker: types.Config{
-			Sizes: &StdSizes{
-				IntSize:  int64(c.targetData.TypeAllocSize(c.intType)),
-				PtrSize:  int64(c.targetData.PointerSize()),
-				MaxAlign: int64(c.targetData.PrefTypeAlignment(c.i8ptrType)),
-			},
-		},
+	wd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	lprogram := &loader.Program{
 		Build: &build.Context{
 			GOARCH:      tripleSplit[0],
 			GOOS:        tripleSplit[2],
@@ -201,15 +197,32 @@ func (c *Compiler) Compile(mainPath string) error {
 			Compiler:    "gc", // must be one of the recognized compilers
 			BuildTags:   append([]string{"tinygo", "gc." + c.selectGC()}, c.BuildTags...),
 		},
-		ParserMode: parser.ParseComments,
+		TypeChecker: types.Config{
+			Sizes: &StdSizes{
+				IntSize:  int64(c.targetData.TypeAllocSize(c.intType)),
+				PtrSize:  int64(c.targetData.PointerSize()),
+				MaxAlign: int64(c.targetData.PrefTypeAlignment(c.i8ptrType)),
+			},
+		},
+		Dir: wd,
 	}
-	config.Import("runtime")
 	if strings.HasSuffix(mainPath, ".go") {
-		config.CreateFromFilenames("main", mainPath)
+		_, err = lprogram.ImportFile(mainPath)
+		if err != nil {
+			return err
+		}
 	} else {
-		config.Import(mainPath)
+		_, err = lprogram.Import(mainPath, wd)
+		if err != nil {
+			return err
+		}
 	}
-	lprogram, err := config.Load()
+	_, err = lprogram.Import("runtime", "")
+	if err != nil {
+		return err
+	}
+
+	err = lprogram.Parse()
 	if err != nil {
 		return err
 	}

ir/ir.go

@@ -8,9 +8,8 @@ import (
 	"strings"
 
 	"github.com/aykevl/go-llvm"
-	"golang.org/x/tools/go/loader"
+	"github.com/aykevl/tinygo/loader"
 	"golang.org/x/tools/go/ssa"
-	"golang.org/x/tools/go/ssa/ssautil"
 )
 
 // This file provides a wrapper around go/ssa values and adds extra
@@ -78,7 +77,7 @@ type Interface struct {
 // Create and intialize a new *Program from a *ssa.Program.
 func NewProgram(lprogram *loader.Program, mainPath string) *Program {
 	comments := map[string]*ast.CommentGroup{}
-	for _, pkgInfo := range lprogram.AllPackages {
+	for _, pkgInfo := range lprogram.Sorted() {
 		for _, file := range pkgInfo.Files {
 			for _, decl := range file.Decls {
 				switch decl := decl.(type) {
@@ -106,7 +105,7 @@ func NewProgram(lprogram *loader.Program, mainPath string) *Program {
 		}
 	}
 
-	program := ssautil.CreateProgram(lprogram, ssa.SanityCheckFunctions|ssa.BareInits|ssa.GlobalDebug)
+	program := lprogram.LoadSSA()
 	program.Build()
 
 	// Find the main package, which is a bit difficult when running a .go file

loader/errors.go

@@ -0,0 +1,27 @@
+package loader
+
+// Errors contains a list of parser errors or a list of typechecker errors for
+// the given package.
+type Errors struct {
+	Pkg  *Package
+	Errs []error
+}
+
+func (e Errors) Error() string {
+	return "could not compile: " + e.Errs[0].Error()
+}
+
+// ImportCycleErrors is returned when encountering an import cycle. The list of
+// packages is a list from the root package to the leaf package that imports one
+// of the packages in the list.
+type ImportCycleError struct {
+	Packages []string
+}
+
+func (e *ImportCycleError) Error() string {
+	msg := "import cycle: " + e.Packages[0]
+	for _, path := range e.Packages[1:] {
+		msg += " → " + path
+	}
+	return msg
+}

loader/loader.go

@@ -0,0 +1,333 @@
+package loader
+
+import (
+	"errors"
+	"go/ast"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"go/types"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// Program holds all packages and some metadata about the program as a whole.
+type Program struct {
+	Build       *build.Context
+	Packages    map[string]*Package
+	sorted      []*Package
+	fset        *token.FileSet
+	TypeChecker types.Config
+	Dir         string // current working directory (for error reporting)
+}
+
+// Package holds a loaded package, its imports, and its parsed files.
+type Package struct {
+	*Program
+	*build.Package
+	Imports   map[string]*Package
+	Importing bool
+	Files     []*ast.File
+	Pkg       *types.Package
+	types.Info
+}
+
+// Import loads the given package relative to srcDir (for the vendor directory).
+// It only loads the current package without recursion.
+func (p *Program) Import(path, srcDir string) (*Package, error) {
+	if p.Packages == nil {
+		p.Packages = make(map[string]*Package)
+	}
+
+	// Load this package.
+	buildPkg, err := p.Build.Import(path, srcDir, build.ImportComment)
+	if err != nil {
+		return nil, err
+	}
+	if existingPkg, ok := p.Packages[buildPkg.ImportPath]; ok {
+		// Already imported, or at least started the import.
+		return existingPkg, nil
+	}
+	p.sorted = nil // invalidate the sorted order of packages
+	pkg := p.newPackage(buildPkg)
+	p.Packages[buildPkg.ImportPath] = pkg
+	return pkg, nil
+}
+
+// ImportFile loads and parses the import statements in the given path and
+// creates a pseudo-package out of it.
+func (p *Program) ImportFile(path string) (*Package, error) {
+	if p.Packages == nil {
+		p.Packages = make(map[string]*Package)
+	}
+	if _, ok := p.Packages[path]; ok {
+		// unlikely
+		return nil, errors.New("loader: cannot import file that is already imported as package: " + path)
+	}
+
+	file, err := p.parseFile(path, parser.ImportsOnly)
+	if err != nil {
+		return nil, err
+	}
+	buildPkg := &build.Package{
+		Dir:        filepath.Dir(path),
+		ImportPath: path,
+		GoFiles:    []string{filepath.Base(path)},
+	}
+	for _, importSpec := range file.Imports {
+		buildPkg.Imports = append(buildPkg.Imports, importSpec.Path.Value[1:len(importSpec.Path.Value)-1])
+	}
+	p.sorted = nil // invalidate the sorted order of packages
+	pkg := p.newPackage(buildPkg)
+	p.Packages[buildPkg.ImportPath] = pkg
+	return pkg, nil
+}
+
+// newPackage instantiates a new *Package object with initialized members.
+func (p *Program) newPackage(pkg *build.Package) *Package {
+	return &Package{
+		Program: p,
+		Package: pkg,
+		Imports: make(map[string]*Package, len(pkg.Imports)),
+		Info: types.Info{
+			Types:      make(map[ast.Expr]types.TypeAndValue),
+			Defs:       make(map[*ast.Ident]types.Object),
+			Uses:       make(map[*ast.Ident]types.Object),
+			Implicits:  make(map[ast.Node]types.Object),
+			Scopes:     make(map[ast.Node]*types.Scope),
+			Selections: make(map[*ast.SelectorExpr]*types.Selection),
+		},
+	}
+}
+
+// Sorted returns a list of all packages, sorted in a way that no packages come
+// before the packages they depend upon.
+func (p *Program) Sorted() []*Package {
+	if p.sorted == nil {
+		p.sort()
+	}
+	return p.sorted
+}
+
+func (p *Program) sort() {
+	p.sorted = nil
+	packageList := make([]*Package, 0, len(p.Packages))
+	packageSet := make(map[string]struct{}, len(p.Packages))
+	worklist := make([]string, 0, len(p.Packages))
+	for path := range p.Packages {
+		worklist = append(worklist, path)
+	}
+	sort.Strings(worklist)
+	for len(worklist) != 0 {
+		pkgPath := worklist[0]
+		pkg := p.Packages[pkgPath]
+
+		if _, ok := packageSet[pkgPath]; ok {
+			// Package already in the final package list.
+			worklist = worklist[1:]
+			continue
+		}
+
+		unsatisfiedImports := make([]string, 0)
+		for _, pkg := range pkg.Imports {
+			if _, ok := packageSet[pkg.ImportPath]; ok {
+				continue
+			}
+			unsatisfiedImports = append(unsatisfiedImports, pkg.ImportPath)
+		}
+		sort.Strings(unsatisfiedImports)
+		if len(unsatisfiedImports) == 0 {
+			// All dependencies of this package are satisfied, so add this
+			// package to the list.
+			packageList = append(packageList, pkg)
+			packageSet[pkgPath] = struct{}{}
+			worklist = worklist[1:]
+		} else {
+			// Prepend all dependencies to the worklist and reconsider this
+			// package (by not removing it from the worklist). At that point, it
+			// must be possible to add it to packageList.
+			worklist = append(unsatisfiedImports, worklist...)
+		}
+	}
+
+	p.sorted = packageList
+}
+
+// Parse recursively imports all packages, parses them, and typechecks them.
+//
+// The returned error may be an Errors error, which contains a list of errors.
+//
+// Idempotent.
+func (p *Program) Parse() error {
+	// Load all imports
+	for _, pkg := range p.Sorted() {
+		err := pkg.importRecursively()
+		if err != nil {
+			if err, ok := err.(*ImportCycleError); ok {
+				err.Packages = append([]string{pkg.ImportPath}, err.Packages...)
+			}
+			return err
+		}
+	}
+
+	// Parse all packages.
+	for _, pkg := range p.Sorted() {
+		err := pkg.Parse()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Typecheck all packages.
+	for _, pkg := range p.Sorted() {
+		err := pkg.Check()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// parseFile is a wrapper around parser.ParseFile.
+func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
+	if p.fset == nil {
+		p.fset = token.NewFileSet()
+	}
+
+	rd, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer rd.Close()
+	relpath := path
+	if filepath.IsAbs(path) {
+		relpath, err = filepath.Rel(p.Dir, path)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return parser.ParseFile(p.fset, relpath, rd, mode)
+}
+
+// Parse parses and typechecks this package.
+//
+// Idempotent.
+func (p *Package) Parse() error {
+	if len(p.Files) != 0 {
+		return nil
+	}
+
+	// Load the AST.
+	// TODO: do this in parallel.
+	if p.ImportPath == "unsafe" {
+		// Special case for the unsafe package. Don't even bother loading
+		// the files.
+		p.Pkg = types.Unsafe
+		return nil
+	}
+
+	files, err := p.parseFiles()
+	if err != nil {
+		return err
+	}
+	p.Files = files
+
+	return nil
+}
+
+// Check runs the package through the typechecker. The package must already be
+// loaded and all dependencies must have been checked already.
+//
+// Idempotent.
+func (p *Package) Check() error {
+	if p.Pkg != nil {
+		return nil
+	}
+
+	var typeErrors []error
+	checker := p.TypeChecker
+	checker.Error = func(err error) {
+		typeErrors = append(typeErrors, err)
+	}
+
+	// Do typechecking of the package.
+	checker.Importer = p
+
+	typesPkg, err := checker.Check(p.ImportPath, p.fset, p.Files, &p.Info)
+	if err != nil {
+		if err, ok := err.(Errors); ok {
+			return err
+		}
+		return Errors{p, typeErrors}
+	}
+	p.Pkg = typesPkg
+	return nil
+}
+
+// parseFiles parses the loaded list of files and returns this list.
+func (p *Package) parseFiles() ([]*ast.File, error) {
+	if len(p.CgoFiles) != 0 {
+		return nil, errors.New("loader: todo cgo: " + p.CgoFiles[0])
+	}
+
+	// TODO: do this concurrently.
+	var files []*ast.File
+	var fileErrs []error
+	for _, file := range p.GoFiles {
+		f, err := p.parseFile(filepath.Join(p.Package.Dir, file), parser.ParseComments)
+		if err != nil {
+			fileErrs = append(fileErrs, err)
+		} else {
+			files = append(files, f)
+		}
+	}
+	if len(fileErrs) != 0 {
+		return nil, Errors{p, fileErrs}
+	}
+	return files, nil
+}
+
+// Import implements types.Importer. It loads and parses packages it encounters
+// along the way, if needed.
+func (p *Package) Import(to string) (*types.Package, error) {
+	if to == "unsafe" {
+		return types.Unsafe, nil
+	}
+	if _, ok := p.Imports[to]; ok {
+		return p.Imports[to].Pkg, nil
+	} else {
+		panic("package not imported: " + to)
+	}
+}
+
+// importRecursively calls Program.Import() on all imported packages, and calls
+// importRecursively() on the imported packages as well.
+//
+// Idempotent.
+func (p *Package) importRecursively() error {
+	p.Importing = true
+	for _, to := range p.Package.Imports {
+		if _, ok := p.Imports[to]; ok {
+			continue
+		}
+		importedPkg, err := p.Program.Import(to, p.Package.Dir)
+		if err != nil {
+			if err, ok := err.(*ImportCycleError); ok {
+				err.Packages = append([]string{p.ImportPath}, err.Packages...)
+			}
+			return err
+		}
+		if importedPkg.Importing {
+			return &ImportCycleError{[]string{p.ImportPath, importedPkg.ImportPath}}
+		}
+		err = importedPkg.importRecursively()
+		if err != nil {
+			return err
+		}
+		p.Imports[to] = importedPkg
+	}
+	p.Importing = false
+	return nil
+}

loader/ssa.go

@@ -0,0 +1,18 @@
+package loader
+
+import (
+	"golang.org/x/tools/go/ssa"
+)
+
+// LoadSSA constructs the SSA form of the loaded packages.
+//
+// The program must already be parsed and type-checked with the .Parse() method.
+func (p *Program) LoadSSA() *ssa.Program {
+	prog := ssa.NewProgram(p.fset, ssa.SanityCheckFunctions|ssa.BareInits|ssa.GlobalDebug)
+
+	for _, pkg := range p.Sorted() {
+		prog.CreatePackage(pkg.Pkg, pkg.Files, &pkg.Info, true)
+	}
+
+	return prog
+}

main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/aykevl/go-llvm"
 	"github.com/aykevl/tinygo/compiler"
 	"github.com/aykevl/tinygo/interp"
+	"github.com/aykevl/tinygo/loader"
 )
 
 var commands = map[string]string{
@@ -454,6 +455,11 @@ func handleCompilerError(err error) {
 			fmt.Fprintln(os.Stderr)
 		} else if errCompiler, ok := err.(types.Error); ok {
 			fmt.Fprintln(os.Stderr, errCompiler)
+		} else if errLoader, ok := err.(loader.Errors); ok {
+			fmt.Fprintln(os.Stderr, "#", errLoader.Pkg.ImportPath)
+			for _, err := range errLoader.Errs {
+				fmt.Fprintln(os.Stderr, err)
+			}
 		} else {
 			fmt.Fprintln(os.Stderr, "error:", err)
 		}