builder: refactor compiler-rt library

This refactor makes adding a new library (such as a libc) much easier in
the future as it avoids a lot of duplicate code. Additionally, CI should
become a little bit faster (~15s) as build-builtins now uses the build
cache.

builder/build.go

@@ -130,21 +130,18 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(stri
 			return err
 		}
 
-		// Load builtins library from the cache, possibly compiling it on the
-		// fly.
-		var librt string
-		if config.Target.RTLib == "compiler-rt" {
-			librt, err = loadBuiltins(config.Triple())
-			if err != nil {
-				return err
-			}
-		}
-
 		// Prepare link command.
 		executable := filepath.Join(dir, "main")
 		tmppath := executable // final file
 		ldflags := append(config.LDFlags(), "-o", executable, objfile)
+
+		// Load builtins library from the cache, possibly compiling it on the
+		// fly.
 		if config.Target.RTLib == "compiler-rt" {
+			librt, err := CompilerRT.Load(config.Triple())
+			if err != nil {
+				return err
+			}
 			ldflags = append(ldflags, librt)
 		}
 

builder/builtins.go

@@ -1,12 +1,7 @@
 package builder
 
 import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"strings"
-
-	"github.com/tinygo-org/tinygo/goenv"
 )
 
 // These are the GENERIC_SOURCES according to CMakeList.txt.
@@ -156,105 +151,20 @@ var aeabiBuiltins = []string{
 	"arm/aeabi_uldivmod.S",
 }
 
-func builtinFiles(target string) []string {
+// CompilerRT is a library with symbols required by programs compiled with LLVM.
-	builtins := append([]string{}, genericBuiltins...) // copy genericBuiltins
+// These symbols are for operations that cannot be emitted with a single
-	if strings.HasPrefix(target, "arm") || strings.HasPrefix(target, "thumb") {
+// instruction or a short sequence of instructions for that target.
-		builtins = append(builtins, aeabiBuiltins...)
+//
-	}
+// For more information, see: https://compiler-rt.llvm.org/
-	return builtins
+var CompilerRT = Library{
-}
+	name:      "compiler-rt",
-
+	cflags:    func() []string { return []string{"-Werror", "-Wall", "-std=c11", "-fshort-enums", "-nostdlibinc"} },
-// builtinsDir returns the directory where the sources for compiler-rt are kept.
+	sourceDir: "lib/compiler-rt/lib/builtins",
-func builtinsDir() string {
+	sources: func(target string) []string {
-	return filepath.Join(goenv.Get("TINYGOROOT"), "lib", "compiler-rt", "lib", "builtins")
+		builtins := append([]string{}, genericBuiltins...) // copy genericBuiltins
-}
+		if strings.HasPrefix(target, "arm") || strings.HasPrefix(target, "thumb") {
-
+			builtins = append(builtins, aeabiBuiltins...)
-// Get the builtins archive, possibly generating it as needed.
-func loadBuiltins(target string) (path string, err error) {
-	// Try to load a precompiled compiler-rt library.
-	precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, "compiler-rt.a")
-	if _, err := os.Stat(precompiledPath); err == nil {
-		// Found a precompiled compiler-rt for this OS/architecture. Return the
-		// path directly.
-		return precompiledPath, nil
-	}
-
-	outfile := "librt-" + target + ".a"
-	builtinsDir := builtinsDir()
-
-	builtins := builtinFiles(target)
-	srcs := make([]string, len(builtins))
-	for i, name := range builtins {
-		srcs[i] = filepath.Join(builtinsDir, name)
-	}
-
-	if path, err := cacheLoad(outfile, commands["clang"][0], srcs); path != "" || err != nil {
-		return path, err
-	}
-
-	var cachepath string
-	err = CompileBuiltins(target, func(path string) error {
-		path, err := cacheStore(path, outfile, commands["clang"][0], srcs)
-		cachepath = path
-		return err
-	})
-	return cachepath, err
-}
-
-// CompileBuiltins compiles builtins from compiler-rt into a static library.
-// When it succeeds, it will call the callback with the resulting path. The path
-// will be removed after callback returns. If callback returns an error, this is
-// passed through to the return value of this function.
-func CompileBuiltins(target string, callback func(path string) error) error {
-	builtinsDir := builtinsDir()
-
-	builtins := builtinFiles(target)
-	srcs := make([]string, len(builtins))
-	for i, name := range builtins {
-		srcs[i] = filepath.Join(builtinsDir, name)
-	}
-
-	dirPrefix := "tinygo-builtins"
-	remapDir := filepath.Join(os.TempDir(), dirPrefix)
-	dir, err := ioutil.TempDir(os.TempDir(), dirPrefix)
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(dir)
-
-	// Compile all builtins.
-	// TODO: use builtins optimized for a given target if available.
-	objs := make([]string, 0, len(builtins))
-	for _, name := range builtins {
-		objname := name
-		if strings.LastIndexByte(objname, '/') >= 0 {
-			objname = objname[strings.LastIndexByte(objname, '/'):]
 		}
-		objpath := filepath.Join(dir, objname+".o")
+		return builtins
-		objs = append(objs, objpath)
+	},
-		srcpath := filepath.Join(builtinsDir, name)
-		// Note: -fdebug-prefix-map is necessary to make the output archive
-		// reproducible. Otherwise the temporary directory is stored in the
-		// archive itself, which varies each run.
-		args := []string{"-c", "-Oz", "-g", "-Werror", "-Wall", "-std=c11", "-fshort-enums", "-nostdlibinc", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target=" + target, "-fdebug-prefix-map=" + dir + "=" + remapDir}
-		if strings.HasPrefix(target, "riscv32-") {
-			args = append(args, "-march=rv32imac", "-mabi=ilp32", "-fforce-enable-int128")
-		}
-		err := runCCompiler("clang", append(args, "-o", objpath, srcpath)...)
-		if err != nil {
-			return &commandError{"failed to build", srcpath, err}
-		}
-	}
-
-	// Put all the object files in a single archive. This archive file will be
-	// used to statically link compiler-rt.
-	arpath := filepath.Join(dir, "librt.a")
-	err = makeArchive(arpath, objs)
-	if err != nil {
-		return err
-	}
-
-	// Give the caller the resulting file. The callback must copy the file,
-	// because after it returns the temporary directory will be removed.
-	return callback(arpath)
 }

builder/library.go

@@ -0,0 +1,99 @@
+package builder
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/tinygo-org/tinygo/goenv"
+)
+
+// Library is a container for information about a single C library, such as a
+// compiler runtime or libc.
+type Library struct {
+	// The library name, such as compiler-rt or picolibc.
+	name string
+
+	cflags func() []string
+
+	// The source directory, relative to TINYGOROOT.
+	sourceDir string
+
+	// The source files, relative to sourceDir.
+	sources func(target string) []string
+}
+
+// fullPath returns the full path to the source directory.
+func (l *Library) fullPath() string {
+	return filepath.Join(goenv.Get("TINYGOROOT"), l.sourceDir)
+}
+
+// sourcePaths returns a slice with the full paths to the source files.
+func (l *Library) sourcePaths(target string) []string {
+	sources := l.sources(target)
+	paths := make([]string, len(sources))
+	for i, name := range sources {
+		paths[i] = filepath.Join(l.fullPath(), name)
+	}
+	return paths
+}
+
+// Load the library archive, possibly generating and caching it if needed.
+func (l *Library) Load(target string) (path string, err error) {
+	// Try to load a precompiled library.
+	precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, l.name+".a")
+	if _, err := os.Stat(precompiledPath); err == nil {
+		// Found a precompiled library for this OS/architecture. Return the path
+		// directly.
+		return precompiledPath, nil
+	}
+
+	outfile := l.name + "-" + target + ".a"
+
+	// Try to fetch this library from the cache.
+	if path, err := cacheLoad(outfile, commands["clang"][0], l.sourcePaths(target)); path != "" || err != nil {
+		// Cache hit.
+		return path, err
+	}
+	// Cache miss, build it now.
+
+	dirPrefix := "tinygo-" + l.name
+	remapDir := filepath.Join(os.TempDir(), dirPrefix)
+	dir, err := ioutil.TempDir(os.TempDir(), dirPrefix)
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(dir)
+
+	// Precalculate the flags to the compiler invocation.
+	args := append(l.cflags(), "-c", "-Oz", "-g", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir)
+	if strings.HasPrefix(target, "riscv32-") {
+		args = append(args, "-march=rv32imac", "-mabi=ilp32", "-fforce-enable-int128")
+	}
+
+	// Compile all sources.
+	var objs []string
+	for _, srcpath := range l.sourcePaths(target) {
+		objpath := filepath.Join(dir, filepath.Base(srcpath)+".o")
+		objs = append(objs, objpath)
+		// Note: -fdebug-prefix-map is necessary to make the output archive
+		// reproducible. Otherwise the temporary directory is stored in the
+		// archive itself, which varies each run.
+		err := runCCompiler("clang", append(args, "-o", objpath, srcpath)...)
+		if err != nil {
+			return "", &commandError{"failed to build", srcpath, err}
+		}
+	}
+
+	// Put all the object files in a single archive. This archive file will be
+	// used to statically link this library.
+	arpath := filepath.Join(dir, l.name+".a")
+	err = makeArchive(arpath, objs)
+	if err != nil {
+		return "", err
+	}
+
+	// Store this archive in the cache.
+	return cacheStore(arpath, outfile, commands["clang"][0], l.sourcePaths(target))
+}

main.go

@@ -50,6 +50,17 @@ func moveFile(src, dst string) error {
 	}
 	// Failed to move, probably a different filesystem.
 	// Do a copy + remove.
+	err = copyFile(src, dst)
+	if err != nil {
+		return err
+	}
+	return os.Remove(src)
+}
+
+// copyFile copies the given file from src to dst. It copies first to a .tmp
+// file which is then moved over a possibly already existing file at the
+// destination.
+func copyFile(src, dst string) error {
 	inf, err := os.Open(src)
 	if err != nil {
 		return err
@@ -785,10 +796,9 @@ func main() {
 		if *target == "" {
 			fmt.Fprintln(os.Stderr, "No target (-target).")
 		}
-		err := builder.CompileBuiltins(*target, func(path string) error {
+		path, err := builder.CompilerRT.Load(*target)
-			return moveFile(path, *outpath)
-		})
 		handleCompilerError(err)
+		copyFile(path, *outpath)
 	case "flash", "gdb":
 		if *outpath != "" {
 			fmt.Fprintln(os.Stderr, "Output cannot be specified with the flash command.")