builder: refactor link file inputs

Add a 'result' member to the compileJob struct which is used by the link
job to get all the paths that should be linked together. This is not yet
necessary (the paths are fixed), but soon the paths are only known after
a linker dependency has run.

builder/build.go

@@ -181,7 +181,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		// The package has not yet been compiled, so create a job to do so.
 		job := &compileJob{
 			description: "compile package " + pkg.ImportPath,
-			run: func() error {
+			run: func(*compileJob) error {
 				// Compile AST to IR. The compiler.CompilePackage function will
 				// build the SSA as needed.
 				mod, errs := compiler.CompilePackage(pkg.ImportPath, pkg, program.Package(pkg.Pkg), machine, compilerConfig, config.DumpSSA())
@@ -250,7 +250,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	programJob := &compileJob{
 		description:  "link+optimize packages (LTO)",
 		dependencies: packageJobs,
-		run: func() error {
+		run: func(*compileJob) error {
 			// Load and link all the bitcode files. This does not yet optimize
 			// anything, it only links the bitcode files together.
 			ctx := llvm.NewContext()
@@ -370,7 +370,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	outputObjectFileJob := &compileJob{
 		description:  "generate output file",
 		dependencies: []*compileJob{programJob},
-		run: func() error {
+		result:       objfile,
+		run: func(*compileJob) error {
 			llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile)
 			if err != nil {
 				return err
@@ -384,40 +385,32 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	linkerDependencies := []*compileJob{outputObjectFileJob}
 	executable := filepath.Join(dir, "main")
 	tmppath := executable // final file
-	ldflags := append(config.LDFlags(), "-o", executable, objfile)
+	ldflags := append(config.LDFlags(), "-o", executable)
 
 	// Add compiler-rt dependency if needed. Usually this is a simple load from
 	// a cache.
 	if config.Target.RTLib == "compiler-rt" {
-		path, job, err := CompilerRT.load(config.Triple(), config.CPU(), dir)
+		job, err := CompilerRT.load(config.Triple(), config.CPU(), dir)
 		if err != nil {
 			return err
 		}
-		if job != nil {
+		jobs = append(jobs, job.dependencies...)
-			// The library was not loaded from cache so needs to be compiled
+		jobs = append(jobs, job)
-			// (and then stored in the cache).
+		linkerDependencies = append(linkerDependencies, job)
-			jobs = append(jobs, job.dependencies...)
-			jobs = append(jobs, job)
-			linkerDependencies = append(linkerDependencies, job)
-		}
-		ldflags = append(ldflags, path)
 	}
 
 	// Add libc dependency if needed.
 	root := goenv.Get("TINYGOROOT")
 	switch config.Target.Libc {
 	case "picolibc":
-		path, job, err := Picolibc.load(config.Triple(), config.CPU(), dir)
+		job, err := Picolibc.load(config.Triple(), config.CPU(), dir)
 		if err != nil {
 			return err
 		}
-		if job != nil {
+		// The library needs to be compiled (cache miss).
-			// The library needs to be compiled (cache miss).
+		jobs = append(jobs, job.dependencies...)
-			jobs = append(jobs, job.dependencies...)
+		jobs = append(jobs, job)
-			jobs = append(jobs, job)
+		linkerDependencies = append(linkerDependencies, job)
-			linkerDependencies = append(linkerDependencies, job)
-		}
-		ldflags = append(ldflags, path)
 	case "wasi-libc":
 		path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a")
 		if _, err := os.Stat(path); os.IsNotExist(err) {
@@ -438,7 +431,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		outpath := filepath.Join(dir, "extra-"+strconv.Itoa(i)+"-"+filepath.Base(path)+".o")
 		job := &compileJob{
 			description: "compile extra file " + path,
-			run: func() error {
+			result:      outpath,
+			run: func(*compileJob) error {
 				err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, abspath)...)
 				if err != nil {
 					return &commandError{"failed to build", path, err}
@@ -448,7 +442,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		}
 		jobs = append(jobs, job)
 		linkerDependencies = append(linkerDependencies, job)
-		ldflags = append(ldflags, outpath)
 	}
 
 	// Add jobs to compile C files in all packages. This is part of CGo.
@@ -460,7 +453,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"."+strconv.Itoa(j)+"-"+filepath.Base(file)+".o")
 			job := &compileJob{
 				description: "compile CGo file " + file,
-				run: func() error {
+				result:      outpath,
+				run: func(*compileJob) error {
 					err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, file)...)
 					if err != nil {
 						return &commandError{"failed to build", file, err}
@@ -470,7 +464,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			}
 			jobs = append(jobs, job)
 			linkerDependencies = append(linkerDependencies, job)
-			ldflags = append(ldflags, outpath)
 		}
 	}
 
@@ -485,7 +478,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	jobs = append(jobs, &compileJob{
 		description:  "link",
 		dependencies: linkerDependencies,
-		run: func() error {
+		run: func(job *compileJob) error {
+			for _, dependency := range job.dependencies {
+				if dependency.result == "" {
+					return errors.New("dependency without result: " + dependency.description)
+				}
+				ldflags = append(ldflags, dependency.result)
+			}
 			err = link(config.Target.Linker, ldflags...)
 			if err != nil {
 				return &commandError{"failed to link", executable, err}

builder/jobs.go

@@ -27,12 +27,23 @@ const (
 type compileJob struct {
 	description  string // description, only used for logging
 	dependencies []*compileJob
-	run          func() error
+	result       string // result (path)
+	run          func(*compileJob) (err error)
 	state        jobState
 	err          error         // error if finished
 	duration     time.Duration // how long it took to run this job (only set after finishing)
 }
 
+// dummyCompileJob returns a new *compileJob that produces an output without
+// doing anything. This can be useful where a *compileJob producing an output is
+// expected but nothing needs to be done, for example for a load from a cache.
+func dummyCompileJob(result string) *compileJob {
+	return &compileJob{
+		description: "<dummy>",
+		result:      result,
+	}
+}
+
 // readyToRun returns whether this job is ready to run: it is itself not yet
 // started and all dependencies are finished.
 func (job *compileJob) readyToRun() bool {
@@ -150,9 +161,11 @@ func nextJob(jobs []*compileJob) *compileJob {
 func jobWorker(workerChan, doneChan chan *compileJob) {
 	for job := range workerChan {
 		start := time.Now()
-		err := job.run()
+		if job.run != nil {
-		if err != nil {
+			err := job.run(job)
-			job.err = err
+			if err != nil {
+				job.err = err
+			}
 		}
 		job.duration = time.Since(start)
 		doneChan <- job

builder/library.go

@@ -42,30 +42,28 @@ func (l *Library) sourcePaths(target string) []string {
 // The resulting file is stored in the provided tmpdir, which is expected to be
 // removed after the Load call.
 func (l *Library) Load(target, tmpdir string) (path string, err error) {
-	path, job, err := l.load(target, "", tmpdir)
+	job, err := l.load(target, "", tmpdir)
 	if err != nil {
 		return "", err
 	}
-	if job != nil {
+	jobs := append([]*compileJob{job}, job.dependencies...)
-		jobs := append([]*compileJob{job}, job.dependencies...)
+	err = runJobs(jobs)
-		err = runJobs(jobs)
+	return job.result, err
-	}
-	return path, err
 }
 
-// load returns a path to the library file for the given target, loading it from
+// load returns a compile job to build this library file for the given target
-// cache if possible. It will return a non-zero compiler job if the library
+// and CPU. It may return a dummy compileJob if the library build is already
-// wasn't cached, this job (and its dependencies) must be run before the library
+// cached. The path is stored as job.result but is only valid if the job and
-// path is valid.
+// job.dependencies have been run.
 // The provided tmpdir will be used to store intermediary files and possibly the
 // output archive file, it is expected to be removed after use.
-func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob, err error) {
+func (l *Library) load(target, cpu, tmpdir string) (job *compileJob, err error) {
 	// Try to load a precompiled library.
 	precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, l.name+".a")
 	if _, err := os.Stat(precompiledPath); err == nil {
 		// Found a precompiled library for this OS/architecture. Return the path
 		// directly.
-		return precompiledPath, nil, nil
+		return dummyCompileJob(precompiledPath), nil
 	}
 
 	var outfile string
@@ -78,7 +76,7 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob
 	// Try to fetch this library from the cache.
 	if path, err := cacheLoad(outfile, l.sourcePaths(target)); path != "" || err != nil {
 		// Cache hit.
-		return path, nil, err
+		return dummyCompileJob(path), nil
 	}
 	// Cache miss, build it now.
 
@@ -86,7 +84,7 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob
 	dir := filepath.Join(tmpdir, "build-lib-"+l.name)
 	err = os.Mkdir(dir, 0777)
 	if err != nil {
-		return "", nil, err
+		return nil, err
 	}
 
 	// Precalculate the flags to the compiler invocation.
@@ -113,7 +111,8 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob
 	arpath := filepath.Join(dir, l.name+".a")
 	job = &compileJob{
 		description: "ar " + l.name + ".a",
-		run: func() error {
+		result:      arpath,
+		run: func(*compileJob) error {
 			// Create an archive of all object files.
 			err := makeArchive(arpath, objs)
 			if err != nil {
@@ -133,7 +132,7 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob
 		objs = append(objs, objpath)
 		job.dependencies = append(job.dependencies, &compileJob{
 			description: "compile " + srcpath,
-			run: func() error {
+			run: func(*compileJob) error {
 				var compileArgs []string
 				compileArgs = append(compileArgs, args...)
 				compileArgs = append(compileArgs, "-o", objpath, srcpath)
@@ -146,5 +145,5 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob
 		})
 	}
 
-	return arpath, job, nil
+	return job, nil
 }