builder: parallelize most of the build

This commit parallelizes almost everything that can currently be
parallelized. With that, it also introduces a framework for easily
parallelizing other parts of the compiler.

Code for baremetal targets already compiles slightly faster because it
can parallelize the compilation of supporting assembly files. However,
the speedup is especially noticeable when libraries (compiler-rt,
picolibc) also need to be compiled: they will be compiled in parallel
next to the Go files using all available cores. On my dual core laptop
(4 cores if you count hyperthreading) this cuts compilation time roughly
in half when compiling something for a Cortex-M board after running
`tinygo clean`.

builder/build.go

@@ -65,95 +65,42 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		return err
 	}
 
-	// Compile AST to IR.
+	// The slice of jobs that orchestrates most of the build.
-	mod, errs := compiler.CompileProgram(pkgName, lprogram, machine, config)
+	// This is somewhat like an in-memory Makefile with each job being a
-	if errs != nil {
+	// Makefile target.
-		return newMultiError(errs)
+	var jobs []*compileJob
-	}
 
-	if config.Options.PrintIR {
+	// Add job to compile and optimize all Go files at once.
-		fmt.Println("; Generated LLVM IR:")
+	// TODO: parallelize this.
-		fmt.Println(mod.String())
+	var mod llvm.Module
-	}
+	var stackSizeLoads []string
-	if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
+	programJob := &compileJob{
-		return errors.New("verification error after IR construction")
+		description: "compile Go files",
-	}
+		run: func() (err error) {
-
+			mod, err = compileWholeProgram(pkgName, config, lprogram, machine)
-	err = interp.Run(mod, config.DumpSSA())
 			if err != nil {
-		return err
+				return
 			}
-	if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
-		return errors.New("verification error after interpreting runtime.initAll")
-	}
-
-	if config.GOOS() != "darwin" {
-		transform.ApplyFunctionSections(mod) // -ffunction-sections
-	}
-
-	// Browsers cannot handle external functions that have type i64 because it
-	// cannot be represented exactly in JavaScript (JS only has doubles). To
-	// keep functions interoperable, pass int64 types as pointers to
-	// stack-allocated values.
-	// Use -wasm-abi=generic to disable this behaviour.
-	if config.WasmAbi() == "js" {
-		err := transform.ExternalInt64AsPtr(mod)
-		if err != nil {
-			return err
-		}
-	}
-
-	// Optimization levels here are roughly the same as Clang, but probably not
-	// exactly.
-	errs = nil
-	switch config.Options.Opt {
-	/*
-		Currently, turning optimizations off causes compile failures.
-		We rely on the optimizer removing some dead symbols.
-		Avoid providing an option that does not work right now.
-		In the future once everything has been fixed we can re-enable this.
-
-		case "none", "0":
-			errs = transform.Optimize(mod, config, 0, 0, 0) // -O0
-	*/
-	case "1":
-		errs = transform.Optimize(mod, config, 1, 0, 0) // -O1
-	case "2":
-		errs = transform.Optimize(mod, config, 2, 0, 225) // -O2
-	case "s":
-		errs = transform.Optimize(mod, config, 2, 1, 225) // -Os
-	case "z":
-		errs = transform.Optimize(mod, config, 2, 2, 5) // -Oz, default
-	default:
-		errs = []error{errors.New("unknown optimization level: -opt=" + config.Options.Opt)}
-	}
-	if len(errs) > 0 {
-		return newMultiError(errs)
-	}
-	if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
-		return errors.New("verification failure after LLVM optimization passes")
-	}
-
-	// LLVM 11 by default tries to emit tail calls (even with the target feature
-	// disabled) unless it is explicitly disabled with a function attribute.
-	// This is a problem, as it tries to emit them and prints an error when it
-	// can't with this feature disabled.
-	// Because as of september 2020 tail calls are not yet widely supported,
-	// they need to be disabled until they are widely supported (at which point
-	// the +tail-call target feautre can be set).
-	if strings.HasPrefix(config.Triple(), "wasm") {
-		transform.DisableTailCalls(mod)
-	}
-
 			// Make sure stack sizes are loaded from a separate section so they can be
 			// modified after linking.
-	var stackSizeLoads []string
 			if config.AutomaticStackSize() {
 				stackSizeLoads = transform.CreateStackSizeLoads(mod, config)
 			}
+			return
+		},
+	}
+	jobs = append(jobs, programJob)
 
-	// Generate output.
+	// Check whether we only need to create an object file.
+	// If so, we don't need to link anything and will be finished quickly.
 	outext := filepath.Ext(outpath)
+	if outext == ".o" || outext == ".bc" || outext == ".ll" {
+		// Run jobs to produce the LLVM module.
+		err := runJobs(jobs)
+		if err != nil {
+			return err
+		}
+		// Generate output.
 		switch outext {
 		case ".o":
 			llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile)
@@ -168,7 +115,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			data := []byte(mod.String())
 			return ioutil.WriteFile(outpath, data, 0666)
 		default:
-		// Act as a compiler driver.
+			panic("unreachable")
+		}
+	}
+
+	// Act as a compiler driver, as we need to produce a complete executable.
+	// First add all jobs necessary to build this object file, then afterwards
+	// run all jobs in parallel as far as possible.
 
 	// Create a temporary directory for intermediary files.
 	dir, err := ioutil.TempDir("", "tinygo")
@@ -177,72 +130,116 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	}
 	defer os.RemoveAll(dir)
 
-		// Write the object file.
+	// Add job to write the output object file.
 	objfile := filepath.Join(dir, "main.o")
+	outputObjectFileJob := &compileJob{
+		description:  "generate output file",
+		dependencies: []*compileJob{programJob},
+		run: func() error {
 			llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile)
 			if err != nil {
 				return err
 			}
-		err = ioutil.WriteFile(objfile, llvmBuf.Bytes(), 0666)
+			return ioutil.WriteFile(objfile, llvmBuf.Bytes(), 0666)
-		if err != nil {
+		},
-			return err
 	}
+	jobs = append(jobs, outputObjectFileJob)
 
 	// Prepare link command.
+	linkerDependencies := []*compileJob{outputObjectFileJob}
 	executable := filepath.Join(dir, "main")
 	tmppath := executable // final file
 	ldflags := append(config.LDFlags(), "-o", executable, objfile)
 
-		// Load builtins library from the cache, possibly compiling it on the
+	// Add compiler-rt dependency if needed. Usually this is a simple load from
-		// fly.
+	// a cache.
 	if config.Target.RTLib == "compiler-rt" {
-			librt, err := CompilerRT.Load(config.Triple())
+		path, job, err := CompilerRT.load(config.Triple(), dir)
 		if err != nil {
 			return err
 		}
-			ldflags = append(ldflags, librt)
+		if job != nil {
+			// The library was not loaded from cache so needs to be compiled
+			// (and then stored in the cache).
+			jobs = append(jobs, job.dependencies...)
+			jobs = append(jobs, job)
+			linkerDependencies = append(linkerDependencies, job)
+		}
+		ldflags = append(ldflags, path)
 	}
 
-		// Add libc.
+	// Add libc dependency if needed.
 	if config.Target.Libc == "picolibc" {
-			libc, err := Picolibc.Load(config.Triple())
+		path, job, err := Picolibc.load(config.Triple(), dir)
 		if err != nil {
 			return err
 		}
-			ldflags = append(ldflags, libc)
+		if job != nil {
+			// The library needs to be compiled (cache miss).
+			jobs = append(jobs, job.dependencies...)
+			jobs = append(jobs, job)
+			linkerDependencies = append(linkerDependencies, job)
+		}
+		ldflags = append(ldflags, path)
 	}
 
-		// Compile extra files.
+	// Add jobs to compile extra files. These files are in C or assembly and
+	// contain things like the interrupt vector table and low level operations
+	// such as stack switching.
 	root := goenv.Get("TINYGOROOT")
 	for i, path := range config.ExtraFiles() {
 		abspath := filepath.Join(root, path)
 		outpath := filepath.Join(dir, "extra-"+strconv.Itoa(i)+"-"+filepath.Base(path)+".o")
+		job := &compileJob{
+			description: "compile extra file " + path,
+			run: func() error {
 				err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, abspath)...)
 				if err != nil {
 					return &commandError{"failed to build", path, err}
 				}
+				return nil
+			},
+		}
+		jobs = append(jobs, job)
+		linkerDependencies = append(linkerDependencies, job)
 		ldflags = append(ldflags, outpath)
 	}
 
-		// Compile C files in packages.
+	// Add jobs to compile C files in all packages. This is part of CGo.
-		// Gather the list of (C) file paths that should be included in the build.
+	// TODO: do this as part of building the package to be able to link the
+	// bitcode files together.
 	for i, pkg := range lprogram.Sorted() {
 		for j, filename := range pkg.CFiles {
 			file := filepath.Join(pkg.Dir, filename)
 			outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"."+strconv.Itoa(j)+"-"+filepath.Base(file)+".o")
+			job := &compileJob{
+				description: "compile CGo file " + file,
+				run: func() error {
 					err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, file)...)
 					if err != nil {
 						return &commandError{"failed to build", file, err}
 					}
+					return nil
+				},
+			}
+			jobs = append(jobs, job)
+			linkerDependencies = append(linkerDependencies, job)
 			ldflags = append(ldflags, outpath)
 		}
 	}
 
+	// Linker flags from CGo lines:
+	//     #cgo LDFLAGS: foo
 	if len(lprogram.LDFlags) > 0 {
 		ldflags = append(ldflags, lprogram.LDFlags...)
 	}
 
-		// Link the object files together.
+	// Create a linker job, which links all object files together and does some
+	// extra stuff that can only be done after linking.
+	jobs = append(jobs, &compileJob{
+		description:  "link",
+		dependencies: linkerDependencies,
+		run: func() error {
 			err = link(config.Target.Linker, ldflags...)
 			if err != nil {
 				return &commandError{"failed to link", executable, err}
@@ -292,6 +289,18 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 				printStacks(calculatedStacks, stackSizes)
 			}
 
+			return nil
+		},
+	})
+
+	// Run all jobs to compile and link the program.
+	// Do this now (instead of after elf-to-hex and similar conversions) as it
+	// is simpler and cannot be parallelized.
+	err = runJobs(jobs)
+	if err != nil {
+		return err
+	}
+
 	// Get an Intel .hex file or .bin file from the .elf file.
 	outputBinaryFormat := config.BinaryFormat(outext)
 	switch outputBinaryFormat {
@@ -328,6 +337,91 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		MainDir: lprogram.MainPkg().Dir,
 	})
 }
+
+// compileWholeProgram compiles the entire *loader.Program to a LLVM module and
+// applies most necessary optimizations and transformations.
+func compileWholeProgram(pkgName string, config *compileopts.Config, lprogram *loader.Program, machine llvm.TargetMachine) (llvm.Module, error) {
+	// Compile AST to IR.
+	mod, errs := compiler.CompileProgram(pkgName, lprogram, machine, config)
+	if errs != nil {
+		return mod, newMultiError(errs)
+	}
+
+	if config.Options.PrintIR {
+		fmt.Println("; Generated LLVM IR:")
+		fmt.Println(mod.String())
+	}
+	if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
+		return mod, errors.New("verification error after IR construction")
+	}
+
+	err := interp.Run(mod, config.DumpSSA())
+	if err != nil {
+		return mod, err
+	}
+	if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
+		return mod, errors.New("verification error after interpreting runtime.initAll")
+	}
+
+	if config.GOOS() != "darwin" {
+		transform.ApplyFunctionSections(mod) // -ffunction-sections
+	}
+
+	// Browsers cannot handle external functions that have type i64 because it
+	// cannot be represented exactly in JavaScript (JS only has doubles). To
+	// keep functions interoperable, pass int64 types as pointers to
+	// stack-allocated values.
+	// Use -wasm-abi=generic to disable this behaviour.
+	if config.WasmAbi() == "js" {
+		err := transform.ExternalInt64AsPtr(mod)
+		if err != nil {
+			return mod, err
+		}
+	}
+
+	// Optimization levels here are roughly the same as Clang, but probably not
+	// exactly.
+	errs = nil
+	switch config.Options.Opt {
+	/*
+		Currently, turning optimizations off causes compile failures.
+		We rely on the optimizer removing some dead symbols.
+		Avoid providing an option that does not work right now.
+		In the future once everything has been fixed we can re-enable this.
+
+		case "none", "0":
+			errs = transform.Optimize(mod, config, 0, 0, 0) // -O0
+	*/
+	case "1":
+		errs = transform.Optimize(mod, config, 1, 0, 0) // -O1
+	case "2":
+		errs = transform.Optimize(mod, config, 2, 0, 225) // -O2
+	case "s":
+		errs = transform.Optimize(mod, config, 2, 1, 225) // -Os
+	case "z":
+		errs = transform.Optimize(mod, config, 2, 2, 5) // -Oz, default
+	default:
+		errs = []error{errors.New("unknown optimization level: -opt=" + config.Options.Opt)}
+	}
+	if len(errs) > 0 {
+		return mod, newMultiError(errs)
+	}
+	if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
+		return mod, errors.New("verification failure after LLVM optimization passes")
+	}
+
+	// LLVM 11 by default tries to emit tail calls (even with the target feature
+	// disabled) unless it is explicitly disabled with a function attribute.
+	// This is a problem, as it tries to emit them and prints an error when it
+	// can't with this feature disabled.
+	// Because as of september 2020 tail calls are not yet widely supported,
+	// they need to be disabled until they are widely supported (at which point
+	// the +tail-call target feautre can be set).
+	if strings.HasPrefix(config.Triple(), "wasm") {
+		transform.DisableTailCalls(mod)
+	}
+
+	return mod, nil
 }
 
 // functionStackSizes keeps stack size information about a single function

builder/buildcache.go

@@ -74,24 +74,16 @@ func cacheStore(tmppath, name, configKey string, sourceFiles []string) (string,
 		return "", err
 	}
 	cachepath := filepath.Join(dir, name)
-	err = moveFile(tmppath, cachepath)
+	err = copyFile(tmppath, cachepath)
 	if err != nil {
 		return "", err
 	}
 	return cachepath, nil
 }
 
-// moveFile renames the file from src to dst. If renaming doesn't work (for
+// copyFile copies the given file from src to dst. It can copy over
-// example, the rename crosses a filesystem boundary), the file is copied and
+// a possibly already existing file at the destination.
-// the old file is removed.
+func copyFile(src, dst string) error {
-func moveFile(src, dst string) error {
-	err := os.Rename(src, dst)
-	if err == nil {
-		// Success!
-		return nil
-	}
-	// Failed to move, probably a different filesystem.
-	// Do a copy + remove.
 	inf, err := os.Open(src)
 	if err != nil {
 		return err

builder/jobs.go

@@ -0,0 +1,154 @@
+package builder
+
+// This file implements a job runner for the compiler, which runs jobs in
+// parallel while taking care of dependencies.
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+)
+
+// Set to true to enable logging in the job runner. This may help to debug
+// concurrency or performance issues.
+const jobRunnerDebug = false
+
+type jobState uint8
+
+const (
+	jobStateQueued   jobState = iota // not yet running
+	jobStateRunning                  // running
+	jobStateFinished                 // finished running
+)
+
+// compileJob is a single compiler job, comparable to a single Makefile target.
+// It is used to orchestrate various compiler tasks that can be run in parallel
+// but that have dependencies and thus have limitations in how they can be run.
+type compileJob struct {
+	description  string // description, only used for logging
+	dependencies []*compileJob
+	run          func() error
+	state        jobState
+	err          error         // error if finished
+	duration     time.Duration // how long it took to run this job (only set after finishing)
+}
+
+// readyToRun returns whether this job is ready to run: it is itself not yet
+// started and all dependencies are finished.
+func (job *compileJob) readyToRun() bool {
+	if job.state != jobStateQueued {
+		// Already running or finished, so shouldn't be run again.
+		return false
+	}
+
+	// Check dependencies.
+	for _, dep := range job.dependencies {
+		if dep.state != jobStateFinished {
+			// A dependency is not finished, so this job has to wait until it
+			// is.
+			return false
+		}
+	}
+
+	// All conditions are satisfied.
+	return true
+}
+
+// runJobs runs all the jobs indicated in the jobs slice and returns the error
+// of the first job that fails to run.
+// It runs all jobs in the order of the slice, as long as all dependencies have
+// already run. Therefore, if some jobs are preferred to run before others, they
+// should be ordered as such in this slice.
+func runJobs(jobs []*compileJob) error {
+	// Create channels to communicate with the workers.
+	doneChan := make(chan *compileJob)
+	workerChan := make(chan *compileJob)
+	defer close(workerChan)
+
+	// Start a number of workers.
+	for i := 0; i < runtime.NumCPU(); i++ {
+		if jobRunnerDebug {
+			fmt.Println("## starting worker", i)
+		}
+		go jobWorker(workerChan, doneChan)
+	}
+
+	// Send each job in the jobs slice to a worker, taking care of job
+	// dependencies.
+	numRunningJobs := 0
+	var totalTime time.Duration
+	start := time.Now()
+	for {
+		// If there are free workers, try starting a new job (if one is
+		// available). If it succeeds, try again to fill the entire worker pool.
+		if numRunningJobs < runtime.NumCPU() {
+			jobToRun := nextJob(jobs)
+			if jobToRun != nil {
+				// Start job.
+				if jobRunnerDebug {
+					fmt.Println("## start:   ", jobToRun.description)
+				}
+				jobToRun.state = jobStateRunning
+				workerChan <- jobToRun
+				numRunningJobs++
+				continue
+			}
+		}
+
+		// When there are no jobs running, all jobs in the jobs slice must have
+		// been finished. Therefore, the work is done.
+		if numRunningJobs == 0 {
+			break
+		}
+
+		// Wait until a job is finished.
+		job := <-doneChan
+		job.state = jobStateFinished
+		numRunningJobs--
+		totalTime += job.duration
+		if jobRunnerDebug {
+			fmt.Println("## finished:", job.description, "(time "+job.duration.String()+")")
+		}
+		if job.err != nil {
+			return job.err
+		}
+	}
+
+	// Some statistics, if debugging.
+	if jobRunnerDebug {
+		// Total duration of running all jobs.
+		duration := time.Since(start)
+		fmt.Println("## total:   ", duration)
+
+		// The individual time of each job combined. On a multicore system, this
+		// should be lower than the total above.
+		fmt.Println("## job sum: ", totalTime)
+	}
+
+	return nil
+}
+
+// nextJob returns the first ready-to-run job.
+// This is an implementation detail of runJobs.
+func nextJob(jobs []*compileJob) *compileJob {
+	for _, job := range jobs {
+		if job.readyToRun() {
+			return job
+		}
+	}
+	return nil
+}
+
+// jobWorker is the goroutine that runs received jobs.
+// This is an implementation detail of runJobs.
+func jobWorker(workerChan, doneChan chan *compileJob) {
+	for job := range workerChan {
+		start := time.Now()
+		err := job.run()
+		if err != nil {
+			job.err = err
+		}
+		job.duration = time.Since(start)
+		doneChan <- job
+	}
+}

builder/library.go

@@ -1,7 +1,6 @@
 package builder
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -40,13 +39,33 @@ func (l *Library) sourcePaths(target string) []string {
 }
 
 // Load the library archive, possibly generating and caching it if needed.
-func (l *Library) Load(target string) (path string, err error) {
+// The resulting file is stored in the provided tmpdir, which is expected to be
+// removed after the Load call.
+func (l *Library) Load(target, tmpdir string) (path string, err error) {
+	path, job, err := l.load(target, tmpdir)
+	if err != nil {
+		return "", err
+	}
+	if job != nil {
+		jobs := append([]*compileJob{job}, job.dependencies...)
+		err = runJobs(jobs)
+	}
+	return path, err
+}
+
+// load returns a path to the library file for the given target, loading it from
+// cache if possible. It will return a non-zero compiler job if the library
+// wasn't cached, this job (and its dependencies) must be run before the library
+// path is valid.
+// The provided tmpdir will be used to store intermediary files and possibly the
+// output archive file, it is expected to be removed after use.
+func (l *Library) load(target, tmpdir string) (path string, job *compileJob, err error) {
 	// Try to load a precompiled library.
 	precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, l.name+".a")
 	if _, err := os.Stat(precompiledPath); err == nil {
 		// Found a precompiled library for this OS/architecture. Return the path
 		// directly.
-		return precompiledPath, nil
+		return precompiledPath, nil, nil
 	}
 
 	outfile := l.name + "-" + target + ".a"
@@ -54,19 +73,21 @@ func (l *Library) Load(target string) (path string, err error) {
 	// Try to fetch this library from the cache.
 	if path, err := cacheLoad(outfile, commands["clang"][0], l.sourcePaths(target)); path != "" || err != nil {
 		// Cache hit.
-		return path, err
+		return path, nil, err
 	}
 	// Cache miss, build it now.
 
-	dirPrefix := "tinygo-" + l.name
+	remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name)
-	remapDir := filepath.Join(os.TempDir(), dirPrefix)
+	dir := filepath.Join(tmpdir, "build-lib-"+l.name)
-	dir, err := ioutil.TempDir(os.TempDir(), dirPrefix)
+	err = os.Mkdir(dir, 0777)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
-	defer os.RemoveAll(dir)
 
 	// Precalculate the flags to the compiler invocation.
+	// Note: -fdebug-prefix-map is necessary to make the output archive
+	// reproducible. Otherwise the temporary directory is stored in the archive
+	// itself, which varies each run.
 	args := append(l.cflags(), "-c", "-Oz", "-g", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir)
 	if strings.HasPrefix(target, "arm") || strings.HasPrefix(target, "thumb") {
 		args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft")
@@ -78,28 +99,44 @@ func (l *Library) Load(target string) (path string, err error) {
 		args = append(args, "-march=rv64gc", "-mabi=lp64")
 	}
 
-	// Compile all sources.
+	// Create job to put all the object files in a single archive. This archive
+	// file is the (static) library file.
 	var objs []string
+	arpath := filepath.Join(dir, l.name+".a")
+	job = &compileJob{
+		description: "ar " + l.name + ".a",
+		run: func() error {
+			// Create an archive of all object files.
+			err := makeArchive(arpath, objs)
+			if err != nil {
+				return err
+			}
+			// Store this archive in the cache.
+			_, err = cacheStore(arpath, outfile, commands["clang"][0], l.sourcePaths(target))
+			return err
+		},
+	}
+
+	// Create jobs to compile all sources. These jobs are depended upon by the
+	// archive job above, so must be run first.
 	for _, srcpath := range l.sourcePaths(target) {
+		srcpath := srcpath // avoid concurrency issues by redefining inside the loop
 		objpath := filepath.Join(dir, filepath.Base(srcpath)+".o")
 		objs = append(objs, objpath)
-		// Note: -fdebug-prefix-map is necessary to make the output archive
+		job.dependencies = append(job.dependencies, &compileJob{
-		// reproducible. Otherwise the temporary directory is stored in the
+			description: "compile " + srcpath,
-		// archive itself, which varies each run.
+			run: func() error {
-		err := runCCompiler("clang", append(args, "-o", objpath, srcpath)...)
+				var compileArgs []string
+				compileArgs = append(compileArgs, args...)
+				compileArgs = append(compileArgs, "-o", objpath, srcpath)
+				err := runCCompiler("clang", compileArgs...)
 				if err != nil {
-			return "", &commandError{"failed to build", srcpath, err}
+					return &commandError{"failed to build", srcpath, err}
 				}
+				return nil
+			},
+		})
 	}
 
-	// Put all the object files in a single archive. This archive file will be
+	return arpath, job, nil
-	// used to statically link this library.
-	arpath := filepath.Join(dir, l.name+".a")
-	err = makeArchive(arpath, objs)
-	if err != nil {
-		return "", err
-	}
-
-	// Store this archive in the cache.
-	return cacheStore(arpath, outfile, commands["clang"][0], l.sourcePaths(target))
 }

main.go

@@ -932,9 +932,17 @@ func main() {
 			fmt.Fprintf(os.Stderr, "Unknown library: %s\n", name)
 			os.Exit(1)
 		}
-		path, err := lib.Load(*target)
+		tmpdir, err := ioutil.TempDir("", "tinygo*")
+		if err != nil {
 			handleCompilerError(err)
-		copyFile(path, outpath)
+		}
+		defer os.RemoveAll(tmpdir)
+		path, err := lib.Load(*target, tmpdir)
+		handleCompilerError(err)
+		err = copyFile(path, outpath)
+		if err != nil {
+			handleCompilerError(err)
+		}
 	case "flash", "gdb":
 		pkgName := filepath.ToSlash(flag.Arg(0))
 		if command == "flash" {