tinygo/loader/loader.go

package loader

import (
	"bytes"
	"errors"
	"go/ast"
	"go/build"
	"go/parser"
	"go/scanner"
	"go/token"
	"go/types"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"text/template"

	"github.com/tinygo-org/tinygo/cgo"
	"github.com/tinygo-org/tinygo/goenv"
)

// Program holds all packages and some metadata about the program as a whole.
type Program struct {
	mainPkg      string
	Build        *build.Context
	Packages     map[string]*Package
	sorted       []*Package
	fset         *token.FileSet
	TypeChecker  types.Config
	Dir          string // current working directory (for error reporting)
	TINYGOROOT   string // root of the TinyGo installation or root of the source code
	CFlags       []string
	LDFlags      []string
	ClangHeaders string
}

// Package holds a loaded package, its imports, and its parsed files.
type Package struct {
	*Program
	*build.Package
	Imports   map[string]*Package
	Importing bool
	Files     []*ast.File
	Pkg       *types.Package
	types.Info
}

// Import loads the given package relative to srcDir (for the vendor directory).
// It only loads the current package without recursion.
func (p *Program) Import(path, srcDir string, pos token.Position) (*Package, error) {
	if p.Packages == nil {
		p.Packages = make(map[string]*Package)
	}

	// Load this package.
	ctx := p.Build
	buildPkg, err := ctx.Import(path, srcDir, build.ImportComment)
	if err != nil {
		return nil, scanner.Error{
			Pos: pos,
			Msg: err.Error(), // TODO: define a new error type that will wrap the inner error
		}
	}
	if existingPkg, ok := p.Packages[buildPkg.ImportPath]; ok {
		// Already imported, or at least started the import.
		return existingPkg, nil
	}
	p.sorted = nil // invalidate the sorted order of packages
	pkg := p.newPackage(buildPkg)
	p.Packages[buildPkg.ImportPath] = pkg

	if p.mainPkg == "" {
		p.mainPkg = buildPkg.ImportPath
	}

	return pkg, nil
}

// ImportFile loads and parses the import statements in the given path and
// creates a pseudo-package out of it.
func (p *Program) ImportFile(path string) (*Package, error) {
	if p.Packages == nil {
		p.Packages = make(map[string]*Package)
	}
	if _, ok := p.Packages[path]; ok {
		// unlikely
		return nil, errors.New("loader: cannot import file that is already imported as package: " + path)
	}

	file, err := p.parseFile(path, parser.ImportsOnly)
	if err != nil {
		return nil, err
	}
	buildPkg := &build.Package{
		Dir:        filepath.Dir(path),
		ImportPath: path,
		GoFiles:    []string{filepath.Base(path)},
	}
	for _, importSpec := range file.Imports {
		buildPkg.Imports = append(buildPkg.Imports, importSpec.Path.Value[1:len(importSpec.Path.Value)-1])
	}
	p.sorted = nil // invalidate the sorted order of packages
	pkg := p.newPackage(buildPkg)
	p.Packages[buildPkg.ImportPath] = pkg

	if p.mainPkg == "" {
		p.mainPkg = buildPkg.ImportPath
	}

	return pkg, nil
}

// newPackage instantiates a new *Package object with initialized members.
func (p *Program) newPackage(pkg *build.Package) *Package {
	return &Package{
		Program: p,
		Package: pkg,
		Imports: make(map[string]*Package, len(pkg.Imports)),
		Info: types.Info{
			Types:      make(map[ast.Expr]types.TypeAndValue),
			Defs:       make(map[*ast.Ident]types.Object),
			Uses:       make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Scopes:     make(map[ast.Node]*types.Scope),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
		},
	}
}

// Sorted returns a list of all packages, sorted in a way that no packages come
// before the packages they depend upon.
func (p *Program) Sorted() []*Package {
	if p.sorted == nil {
		p.sort()
	}
	return p.sorted
}

func (p *Program) sort() {
	p.sorted = nil
	packageList := make([]*Package, 0, len(p.Packages))
	packageSet := make(map[string]struct{}, len(p.Packages))
	worklist := make([]string, 0, len(p.Packages))
	for path := range p.Packages {
		worklist = append(worklist, path)
	}
	sort.Strings(worklist)
	for len(worklist) != 0 {
		pkgPath := worklist[0]
		pkg := p.Packages[pkgPath]

		if _, ok := packageSet[pkgPath]; ok {
			// Package already in the final package list.
			worklist = worklist[1:]
			continue
		}

		unsatisfiedImports := make([]string, 0)
		for _, pkg := range pkg.Imports {
			if _, ok := packageSet[pkg.ImportPath]; ok {
				continue
			}
			unsatisfiedImports = append(unsatisfiedImports, pkg.ImportPath)
		}
		sort.Strings(unsatisfiedImports)
		if len(unsatisfiedImports) == 0 {
			// All dependencies of this package are satisfied, so add this
			// package to the list.
			packageList = append(packageList, pkg)
			packageSet[pkgPath] = struct{}{}
			worklist = worklist[1:]
		} else {
			// Prepend all dependencies to the worklist and reconsider this
			// package (by not removing it from the worklist). At that point, it
			// must be possible to add it to packageList.
			worklist = append(unsatisfiedImports, worklist...)
		}
	}

	p.sorted = packageList
}

// Parse recursively imports all packages, parses them, and typechecks them.
//
// The returned error may be an Errors error, which contains a list of errors.
//
// Idempotent.
func (p *Program) Parse(compileTestBinary bool) error {
	includeTests := compileTestBinary

	// Load all imports
	for _, pkg := range p.Sorted() {
		err := pkg.importRecursively(includeTests)
		if err != nil {
			if err, ok := err.(*ImportCycleError); ok {
				if pkg.ImportPath != err.Packages[0] {
					err.Packages = append([]string{pkg.ImportPath}, err.Packages...)
				}
			}
			return err
		}
	}

	// Parse all packages.
	for _, pkg := range p.Sorted() {
		err := pkg.Parse(includeTests)
		if err != nil {
			return err
		}
	}

	if compileTestBinary {
		err := p.SwapTestMain()
		if err != nil {
			return err
		}
	}

	// Typecheck all packages.
	for _, pkg := range p.Sorted() {
		err := pkg.Check()
		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Program) SwapTestMain() error {
	var tests []string

	isTestFunc := func(f *ast.FuncDecl) bool {
		// TODO: improve signature check
		if strings.HasPrefix(f.Name.Name, "Test") && f.Name.Name != "TestMain" {
			return true
		}
		return false
	}
	mainPkg := p.Packages[p.mainPkg]
	for _, f := range mainPkg.Files {
		for i, d := range f.Decls {
			switch v := d.(type) {
			case *ast.FuncDecl:
				if isTestFunc(v) {
					tests = append(tests, v.Name.Name)
				}
				if v.Name.Name == "main" {
					// Remove main
					if len(f.Decls) == 1 {
						f.Decls = make([]ast.Decl, 0)
					} else {
						f.Decls[i] = f.Decls[len(f.Decls)-1]
						f.Decls = f.Decls[:len(f.Decls)-1]
					}
				}
			}
		}
	}

	// TODO: Check if they defined a TestMain and call it instead of testing.TestMain
	const mainBody = `package main

import (
	"testing"
)

func main () {
	m := &testing.M{
		Tests: []testing.TestToCall{
{{range .TestFunctions}}
			{Name: "{{.}}", Func: {{.}}},
{{end}}
		},
	}

	testing.TestMain(m)
}
`
	tmpl := template.Must(template.New("testmain").Parse(mainBody))
	b := bytes.Buffer{}
	tmplData := struct {
		TestFunctions []string
	}{
		TestFunctions: tests,
	}

	err := tmpl.Execute(&b, tmplData)
	if err != nil {
		return err
	}
	path := filepath.Join(p.mainPkg, "$testmain.go")

	if p.fset == nil {
		p.fset = token.NewFileSet()
	}

	newMain, err := parser.ParseFile(p.fset, path, b.Bytes(), parser.AllErrors)
	if err != nil {
		return err
	}
	mainPkg.Files = append(mainPkg.Files, newMain)

	return nil
}

// parseFile is a wrapper around parser.ParseFile.
func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
	if p.fset == nil {
		p.fset = token.NewFileSet()
	}

	rd, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer rd.Close()
	diagnosticPath := path
	if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) {
		// If this file is part of the synthetic GOROOT, try to infer the
		// original path.
		relpath := path[len(filepath.Join(p.Build.GOROOT, "src"))+1:]
		realgorootPath := filepath.Join(goenv.Get("GOROOT"), "src", relpath)
		if _, err := os.Stat(realgorootPath); err == nil {
			diagnosticPath = realgorootPath
		}
		maybeInTinyGoRoot := false
		for prefix := range pathsToOverride(needsSyscallPackage(p.Build.BuildTags)) {
			if !strings.HasPrefix(relpath, prefix) {
				continue
			}
			maybeInTinyGoRoot = true
		}
		if maybeInTinyGoRoot {
			tinygoPath := filepath.Join(p.TINYGOROOT, "src", relpath)
			if _, err := os.Stat(tinygoPath); err == nil {
				diagnosticPath = tinygoPath
			}
		}
	}
	return parser.ParseFile(p.fset, diagnosticPath, rd, mode)
}

// Parse parses and typechecks this package.
//
// Idempotent.
func (p *Package) Parse(includeTests bool) error {
	if len(p.Files) != 0 {
		return nil
	}

	// Load the AST.
	// TODO: do this in parallel.
	if p.ImportPath == "unsafe" {
		// Special case for the unsafe package. Don't even bother loading
		// the files.
		p.Pkg = types.Unsafe
		return nil
	}

	files, err := p.parseFiles(includeTests)
	if err != nil {
		return err
	}
	p.Files = files

	return nil
}

// Check runs the package through the typechecker. The package must already be
// loaded and all dependencies must have been checked already.
//
// Idempotent.
func (p *Package) Check() error {
	if p.Pkg != nil {
		return nil
	}

	var typeErrors []error
	checker := p.TypeChecker
	checker.Error = func(err error) {
		typeErrors = append(typeErrors, err)
	}

	// Do typechecking of the package.
	checker.Importer = p

	typesPkg, err := checker.Check(p.ImportPath, p.fset, p.Files, &p.Info)
	if err != nil {
		if err, ok := err.(Errors); ok {
			return err
		}
		return Errors{p, typeErrors}
	}
	p.Pkg = typesPkg
	return nil
}

// parseFiles parses the loaded list of files and returns this list.
func (p *Package) parseFiles(includeTests bool) ([]*ast.File, error) {
	// TODO: do this concurrently.
	var files []*ast.File
	var fileErrs []error

	var gofiles []string
	if includeTests {
		gofiles = make([]string, 0, len(p.GoFiles)+len(p.TestGoFiles))
		gofiles = append(gofiles, p.GoFiles...)
		gofiles = append(gofiles, p.TestGoFiles...)
	} else {
		gofiles = p.GoFiles
	}

	for _, file := range gofiles {
		f, err := p.parseFile(filepath.Join(p.Package.Dir, file), parser.ParseComments)
		if err != nil {
			fileErrs = append(fileErrs, err)
			continue
		}
		if err != nil {
			fileErrs = append(fileErrs, err)
			continue
		}
		files = append(files, f)
	}
	for _, file := range p.CgoFiles {
		path := filepath.Join(p.Package.Dir, file)
		f, err := p.parseFile(path, parser.ParseComments)
		if err != nil {
			fileErrs = append(fileErrs, err)
			continue
		}
		files = append(files, f)
	}
	if len(p.CgoFiles) != 0 {
		cflags := append(p.CFlags, "-I"+p.Package.Dir)
		if p.ClangHeaders != "" {
			cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.ClangHeaders)
		}
		generated, ldflags, errs := cgo.Process(files, p.Program.Dir, p.fset, cflags)
		if errs != nil {
			fileErrs = append(fileErrs, errs...)
		}
		files = append(files, generated)
		p.LDFlags = append(p.LDFlags, ldflags...)
	}
	if len(fileErrs) != 0 {
		return nil, Errors{p, fileErrs}
	}

	return files, nil
}

// Import implements types.Importer. It loads and parses packages it encounters
// along the way, if needed.
func (p *Package) Import(to string) (*types.Package, error) {
	if to == "unsafe" {
		return types.Unsafe, nil
	}
	if _, ok := p.Imports[to]; ok {
		return p.Imports[to].Pkg, nil
	} else {
		return nil, errors.New("package not imported: " + to)
	}
}

// importRecursively calls Program.Import() on all imported packages, and calls
// importRecursively() on the imported packages as well.
//
// Idempotent.
func (p *Package) importRecursively(includeTests bool) error {
	p.Importing = true

	imports := p.Package.Imports
	if includeTests {
		imports = append(imports, p.Package.TestImports...)
	}

	for _, to := range imports {
		if to == "C" {
			// Do CGo processing in a later stage.
			continue
		}
		if _, ok := p.Imports[to]; ok {
			continue
		}
		// Find error location.
		var pos token.Position
		if len(p.Package.ImportPos[to]) > 0 {
			pos = p.Package.ImportPos[to][0]
		} else {
			pos = token.Position{Filename: p.Package.ImportPath}
		}
		importedPkg, err := p.Program.Import(to, p.Package.Dir, pos)
		if err != nil {
			if err, ok := err.(*ImportCycleError); ok {
				err.Packages = append([]string{p.ImportPath}, err.Packages...)
			}
			return err
		}
		if importedPkg.Importing {
			return &ImportCycleError{[]string{p.ImportPath, importedPkg.ImportPath}, p.ImportPos[to]}
		}
		err = importedPkg.importRecursively(false)
		if err != nil {
			if err, ok := err.(*ImportCycleError); ok {
				err.Packages = append([]string{p.ImportPath}, err.Packages...)
			}
			return err
		}
		p.Imports[to] = importedPkg
	}
	p.Importing = false
	return nil
}