builder: run interp per package

This results in a significant speedup in some cases. For example, this
runs over twice as fast with a warm cache:

    tinygo build -o test.elf ./testdata/stdlib.go

This should help a lot with edit-compile-test cycles, that typically
only modify a single package.

This required some changes to the interp package to deal with globals
created in a previous run of the interp package and to deal with
external globals (that can't be loaded from or stored to).

builder/build.go

@@ -55,6 +55,7 @@ type BuildResult struct {
 type packageAction struct {
 	ImportPath      string
 	CompilerVersion int // compiler.Version
+	InterpVersion   int // interp.Version
 	LLVMVersion     string
 	Config          *compiler.Config
 	CFlags          []string
@@ -135,6 +136,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		actionID := packageAction{
 			ImportPath:      pkg.ImportPath,
 			CompilerVersion: compiler.Version,
+			InterpVersion:   interp.Version,
 			LLVMVersion:     llvm.Version,
 			Config:          compilerConfig,
 			CFlags:          pkg.CFlags,
@@ -190,6 +192,21 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 					return errors.New("verification error after compiling package " + pkg.ImportPath)
 				}
 
+				// Try to interpret package initializers at compile time.
+				// It may only be possible to do this partially, in which case
+				// it is completed after all IR files are linked.
+				pkgInit := mod.NamedFunction(pkg.Pkg.Path() + ".init")
+				if pkgInit.IsNil() {
+					panic("init not found for " + pkg.Pkg.Path())
+				}
+				err := interp.RunFunc(pkgInit, config.DumpSSA())
+				if err != nil {
+					return err
+				}
+				if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
+					return errors.New("verification error after interpreting " + pkgInit.Name())
+				}
+
 				// Serialize the LLVM module as a bitcode file.
 				// Write to a temporary path that is renamed to the destination
 				// file to avoid race conditions with other TinyGo invocatiosn
@@ -575,9 +592,18 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error {
 	if err != nil {
 		return err
 	}
+	if config.VerifyIR() {
+		// Only verify if we really need it.
+		// The IR has already been verified before writing the bitcode to disk
+		// and the interp function above doesn't need to do a lot as most of the
+		// package initializers have already run. Additionally, verifying this
+		// linked IR is _expensive_ because dead code hasn't been removed yet,
+		// easily costing a few hundred milliseconds. Therefore, only do it when
+		// specifically requested.
 		if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
 			return errors.New("verification error after interpreting runtime.initAll")
 		}
+	}
 
 	if config.GOOS() != "darwin" {
 		transform.ApplyFunctionSections(mod) // -ffunction-sections

interp/errors.go

@@ -20,6 +20,11 @@ var (
 	errMapAlreadyCreated      = errors.New("interp: map already created")
 )
 
+// This is one of the errors that can be returned from toLLVMValue when the
+// passed type does not fit the data to serialize. It is recoverable by
+// serializing without a type (using rawValue.rawLLVMValue).
+var errInvalidPtrToIntSize = errors.New("interp: ptrtoint integer size does not equal pointer size")
+
 func isRecoverableError(err error) bool {
 	return err == errIntegerAsPointer || err == errUnsupportedInst || err == errUnsupportedRuntimeInst || err == errMapAlreadyCreated
 }

interp/interp.go

@@ -11,6 +11,12 @@ import (
 	"tinygo.org/x/go-llvm"
 )
 
+// Version of the interp package. It must be incremented whenever the interp
+// package is changed in a way that affects the output so that cached package
+// builds will be invalidated.
+// This version is independent of the TinyGo version number.
+const Version = 1
+
 // Enable extra checks, which should be disabled by default.
 // This may help track down bugs by adding a few more sanity checks.
 const checks = true
@@ -32,9 +38,7 @@ type runner struct {
 	callsExecuted uint64
 }
 
-// Run evaluates runtime.initAll function as much as possible at compile time.
+func newRunner(mod llvm.Module, debug bool) *runner {
-// Set debug to true if it should print output while running.
-func Run(mod llvm.Module, debug bool) error {
 	r := runner{
 		mod:           mod,
 		targetData:    llvm.NewTargetData(mod.DataLayout()),
@@ -47,6 +51,13 @@ func Run(mod llvm.Module, debug bool) error {
 	r.pointerSize = uint32(r.targetData.PointerSize())
 	r.i8ptrType = llvm.PointerType(mod.Context().Int8Type(), 0)
 	r.maxAlign = r.targetData.PrefTypeAlignment(r.i8ptrType) // assume pointers are maximally aligned (this is not always the case)
+	return &r
+}
+
+// Run evaluates runtime.initAll function as much as possible at compile time.
+// Set debug to true if it should print output while running.
+func Run(mod llvm.Module, debug bool) error {
+	r := newRunner(mod, debug)
 
 	initAll := mod.NamedFunction("runtime.initAll")
 	bb := initAll.EntryBasicBlock()
@@ -117,7 +128,104 @@ func Run(mod llvm.Module, debug bool) error {
 	r.pkgName = ""
 
 	// Update all global variables in the LLVM module.
-	mem := memoryView{r: &r}
+	mem := memoryView{r: r}
+	for _, obj := range r.objects {
+		if obj.llvmGlobal.IsNil() {
+			continue
+		}
+		if obj.buffer == nil {
+			continue
+		}
+		initializer, err := obj.buffer.toLLVMValue(obj.llvmGlobal.Type().ElementType(), &mem)
+		if err == errInvalidPtrToIntSize {
+			// This can happen when a previous interp run did not have the
+			// correct LLVM type for a global and made something up. In that
+			// case, some fields could be written out as a series of (null)
+			// bytes even though they actually contain a pointer value.
+			// As a fallback, use asRawValue to get something of the correct
+			// memory layout.
+			initializer, err := obj.buffer.asRawValue(r).rawLLVMValue(&mem)
+			if err != nil {
+				return err
+			}
+			initializerType := initializer.Type()
+			newGlobal := llvm.AddGlobal(mod, initializerType, obj.llvmGlobal.Name()+".tmp")
+			newGlobal.SetInitializer(initializer)
+			newGlobal.SetLinkage(obj.llvmGlobal.Linkage())
+			newGlobal.SetAlignment(obj.llvmGlobal.Alignment())
+			// TODO: copy debug info, unnamed_addr, ...
+			bitcast := llvm.ConstBitCast(newGlobal, obj.llvmGlobal.Type())
+			obj.llvmGlobal.ReplaceAllUsesWith(bitcast)
+			name := obj.llvmGlobal.Name()
+			obj.llvmGlobal.EraseFromParentAsGlobal()
+			newGlobal.SetName(name)
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		if checks && initializer.Type() != obj.llvmGlobal.Type().ElementType() {
+			panic("initializer type mismatch")
+		}
+		obj.llvmGlobal.SetInitializer(initializer)
+	}
+
+	return nil
+}
+
+// RunFunc evaluates a single package initializer at compile time.
+// Set debug to true if it should print output while running.
+func RunFunc(fn llvm.Value, debug bool) error {
+	// Create and initialize *runner object.
+	mod := fn.GlobalParent()
+	r := newRunner(mod, debug)
+	initName := fn.Name()
+	if !strings.HasSuffix(initName, ".init") {
+		return errorAt(fn, "interp: unexpected function name (expected *.init)")
+	}
+	r.pkgName = initName[:len(initName)-len(".init")]
+
+	// Create new function with the interp result.
+	newFn := llvm.AddFunction(mod, fn.Name()+".tmp", fn.Type().ElementType())
+	newFn.SetLinkage(fn.Linkage())
+	newFn.SetVisibility(fn.Visibility())
+	entry := mod.Context().AddBasicBlock(newFn, "entry")
+
+	// Create a builder, to insert instructions that could not be evaluated at
+	// compile time.
+	r.builder = mod.Context().NewBuilder()
+	defer r.builder.Dispose()
+	r.builder.SetInsertPointAtEnd(entry)
+
+	// Copy debug information.
+	subprogram := fn.Subprogram()
+	if !subprogram.IsNil() {
+		newFn.SetSubprogram(subprogram)
+		r.builder.SetCurrentDebugLocation(subprogram.SubprogramLine(), 0, subprogram, llvm.Metadata{})
+	}
+
+	// Run the initializer, filling the .init.tmp function.
+	if r.debug {
+		fmt.Fprintln(os.Stderr, "interp:", fn.Name())
+	}
+	_, pkgMem, callErr := r.run(r.getFunction(fn), nil, nil, "    ")
+	if callErr != nil {
+		if isRecoverableError(callErr.Err) {
+			// Could not finish, but could recover from it.
+			if r.debug {
+				fmt.Fprintln(os.Stderr, "not interpreting", r.pkgName, "because of error:", callErr.Error())
+			}
+			newFn.EraseFromParentAsFunction()
+			return nil
+		}
+		return callErr
+	}
+	for index, obj := range pkgMem.objects {
+		r.objects[index] = obj
+	}
+
+	// Update globals with values determined while running the initializer above.
+	mem := memoryView{r: r}
 	for _, obj := range r.objects {
 		if obj.llvmGlobal.IsNil() {
 			continue
@@ -135,6 +243,14 @@ func Run(mod llvm.Module, debug bool) error {
 		obj.llvmGlobal.SetInitializer(initializer)
 	}
 
+	// Finalize: remove the old init function and replace it with the new
+	// (.init.tmp) function.
+	r.builder.CreateRetVoid()
+	fnName := fn.Name()
+	fn.ReplaceAllUsesWith(newFn)
+	fn.EraseFromParentAsFunction()
+	newFn.SetName(fnName)
+
 	return nil
 }
 

interp/interpreter.go

@@ -548,6 +548,13 @@ func (r *runner) run(fn *function, params []value, parentMem *memoryView, indent
 				continue
 			}
 			result := mem.load(ptr, uint32(size))
+			if result == nil {
+				err := r.runAtRuntime(fn, inst, locals, &mem, indent)
+				if err != nil {
+					return nil, mem, err
+				}
+				continue
+			}
 			if r.debug {
 				fmt.Fprintln(os.Stderr, indent+"load:", ptr, "->", result)
 			}
@@ -570,7 +577,14 @@ func (r *runner) run(fn *function, params []value, parentMem *memoryView, indent
 			if r.debug {
 				fmt.Fprintln(os.Stderr, indent+"store:", val, ptr)
 			}
-			mem.store(val, ptr)
+			ok := mem.store(val, ptr)
+			if !ok {
+				// Could not store the value, do it at runtime.
+				err := r.runAtRuntime(fn, inst, locals, &mem, indent)
+				if err != nil {
+					return nil, mem, err
+				}
+			}
 		case llvm.Alloca:
 			// Alloca normally allocates some stack memory. In the interpreter,
 			// it allocates a global instead.

interp/memory.go

@@ -259,12 +259,17 @@ func (mv *memoryView) put(index uint32, obj object) {
 	mv.objects[index] = obj
 }
 
-// Load the value behind the given pointer.
+// Load the value behind the given pointer. Returns nil if the pointer points to
+// an external global.
 func (mv *memoryView) load(p pointerValue, size uint32) value {
 	if checks && mv.hasExternalStore(p) {
 		panic("interp: load from object with external store")
 	}
 	obj := mv.get(p.index())
+	if obj.buffer == nil {
+		// External global, return nil.
+		return nil
+	}
 	if p.offset() == 0 && size == obj.size {
 		return obj.buffer.clone()
 	}
@@ -280,12 +285,17 @@ func (mv *memoryView) load(p pointerValue, size uint32) value {
 
 // Store to the value behind the given pointer. This overwrites the value in the
 // memory view, so that the changed value is discarded when the memory view is
-// reverted.
+// reverted. Returns true on success, false if the object to store to is
-func (mv *memoryView) store(v value, p pointerValue) {
+// external.
+func (mv *memoryView) store(v value, p pointerValue) bool {
 	if checks && mv.hasExternalLoadOrStore(p) {
 		panic("interp: store to object with external load/store")
 	}
 	obj := mv.get(p.index())
+	if obj.buffer == nil {
+		// External global, return false (for a failure).
+		return false
+	}
 	if checks && p.offset()+v.len(mv.r) > obj.size {
 		panic("interp: store out of bounds")
 	}
@@ -301,6 +311,7 @@ func (mv *memoryView) store(v value, p pointerValue) {
 		}
 	}
 	mv.put(p.index(), obj)
+	return true // success
 }
 
 // value is some sort of value, comparable to a LLVM constant. It can be
@@ -1105,6 +1116,12 @@ func (v rawValue) toLLVMValue(llvmType llvm.Type, mem *memoryView) (llvm.Value,
 			if err != nil {
 				panic(err)
 			}
+			if checks && mem.r.targetData.TypeAllocSize(llvmType) != mem.r.targetData.TypeAllocSize(mem.r.i8ptrType) {
+				// Probably trying to serialize a pointer to a byte array,
+				// perhaps as a result of rawLLVMValue() in a previous interp
+				// run.
+				return llvm.Value{}, errInvalidPtrToIntSize
+			}
 			v, err := ptr.toLLVMValue(llvm.Type{}, mem)
 			if err != nil {
 				return llvm.Value{}, err