From 83a949647f57316ba3de8136ec9793541291d0e1 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 11 Mar 2021 22:23:32 +0100 Subject: [PATCH] builder: refactor link file inputs Add a 'result' member to the compileJob struct which is used by the link job to get all the paths that should be linked together. This is not yet necessary (the paths are fixed), but soon the paths are only known after a linker dependency has run. --- builder/build.go | 51 +++++++++++++++++++++++----------------------- builder/jobs.go | 21 +++++++++++++++---- builder/library.go | 33 +++++++++++++++--------------- 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/builder/build.go b/builder/build.go index a3d99ec2..1da4d80b 100644 --- a/builder/build.go +++ b/builder/build.go @@ -181,7 +181,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil // The package has not yet been compiled, so create a job to do so. job := &compileJob{ description: "compile package " + pkg.ImportPath, - run: func() error { + run: func(*compileJob) error { // Compile AST to IR. The compiler.CompilePackage function will // build the SSA as needed. mod, errs := compiler.CompilePackage(pkg.ImportPath, pkg, program.Package(pkg.Pkg), machine, compilerConfig, config.DumpSSA()) @@ -250,7 +250,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil programJob := &compileJob{ description: "link+optimize packages (LTO)", dependencies: packageJobs, - run: func() error { + run: func(*compileJob) error { // Load and link all the bitcode files. This does not yet optimize // anything, it only links the bitcode files together. ctx := llvm.NewContext() @@ -370,7 +370,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil outputObjectFileJob := &compileJob{ description: "generate output file", dependencies: []*compileJob{programJob}, - run: func() error { + result: objfile, + run: func(*compileJob) error { llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile) if err != nil { return err @@ -384,40 +385,32 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil linkerDependencies := []*compileJob{outputObjectFileJob} executable := filepath.Join(dir, "main") tmppath := executable // final file - ldflags := append(config.LDFlags(), "-o", executable, objfile) + ldflags := append(config.LDFlags(), "-o", executable) // Add compiler-rt dependency if needed. Usually this is a simple load from // a cache. if config.Target.RTLib == "compiler-rt" { - path, job, err := CompilerRT.load(config.Triple(), config.CPU(), dir) + job, err := CompilerRT.load(config.Triple(), config.CPU(), dir) if err != nil { return err } - if job != nil { - // The library was not loaded from cache so needs to be compiled - // (and then stored in the cache). - jobs = append(jobs, job.dependencies...) - jobs = append(jobs, job) - linkerDependencies = append(linkerDependencies, job) - } - ldflags = append(ldflags, path) + jobs = append(jobs, job.dependencies...) + jobs = append(jobs, job) + linkerDependencies = append(linkerDependencies, job) } // Add libc dependency if needed. root := goenv.Get("TINYGOROOT") switch config.Target.Libc { case "picolibc": - path, job, err := Picolibc.load(config.Triple(), config.CPU(), dir) + job, err := Picolibc.load(config.Triple(), config.CPU(), dir) if err != nil { return err } - if job != nil { - // The library needs to be compiled (cache miss). - jobs = append(jobs, job.dependencies...) - jobs = append(jobs, job) - linkerDependencies = append(linkerDependencies, job) - } - ldflags = append(ldflags, path) + // The library needs to be compiled (cache miss). + jobs = append(jobs, job.dependencies...) + jobs = append(jobs, job) + linkerDependencies = append(linkerDependencies, job) case "wasi-libc": path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a") if _, err := os.Stat(path); os.IsNotExist(err) { @@ -438,7 +431,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil outpath := filepath.Join(dir, "extra-"+strconv.Itoa(i)+"-"+filepath.Base(path)+".o") job := &compileJob{ description: "compile extra file " + path, - run: func() error { + result: outpath, + run: func(*compileJob) error { err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, abspath)...) if err != nil { return &commandError{"failed to build", path, err} @@ -448,7 +442,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil } jobs = append(jobs, job) linkerDependencies = append(linkerDependencies, job) - ldflags = append(ldflags, outpath) } // Add jobs to compile C files in all packages. This is part of CGo. @@ -460,7 +453,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"."+strconv.Itoa(j)+"-"+filepath.Base(file)+".o") job := &compileJob{ description: "compile CGo file " + file, - run: func() error { + result: outpath, + run: func(*compileJob) error { err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, file)...) if err != nil { return &commandError{"failed to build", file, err} @@ -470,7 +464,6 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil } jobs = append(jobs, job) linkerDependencies = append(linkerDependencies, job) - ldflags = append(ldflags, outpath) } } @@ -485,7 +478,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil jobs = append(jobs, &compileJob{ description: "link", dependencies: linkerDependencies, - run: func() error { + run: func(job *compileJob) error { + for _, dependency := range job.dependencies { + if dependency.result == "" { + return errors.New("dependency without result: " + dependency.description) + } + ldflags = append(ldflags, dependency.result) + } err = link(config.Target.Linker, ldflags...) if err != nil { return &commandError{"failed to link", executable, err} diff --git a/builder/jobs.go b/builder/jobs.go index a7bf5ec0..2bc799d3 100644 --- a/builder/jobs.go +++ b/builder/jobs.go @@ -27,12 +27,23 @@ const ( type compileJob struct { description string // description, only used for logging dependencies []*compileJob - run func() error + result string // result (path) + run func(*compileJob) (err error) state jobState err error // error if finished duration time.Duration // how long it took to run this job (only set after finishing) } +// dummyCompileJob returns a new *compileJob that produces an output without +// doing anything. This can be useful where a *compileJob producing an output is +// expected but nothing needs to be done, for example for a load from a cache. +func dummyCompileJob(result string) *compileJob { + return &compileJob{ + description: "", + result: result, + } +} + // readyToRun returns whether this job is ready to run: it is itself not yet // started and all dependencies are finished. func (job *compileJob) readyToRun() bool { @@ -150,9 +161,11 @@ func nextJob(jobs []*compileJob) *compileJob { func jobWorker(workerChan, doneChan chan *compileJob) { for job := range workerChan { start := time.Now() - err := job.run() - if err != nil { - job.err = err + if job.run != nil { + err := job.run(job) + if err != nil { + job.err = err + } } job.duration = time.Since(start) doneChan <- job diff --git a/builder/library.go b/builder/library.go index 40699491..0ce4a6fe 100644 --- a/builder/library.go +++ b/builder/library.go @@ -42,30 +42,28 @@ func (l *Library) sourcePaths(target string) []string { // The resulting file is stored in the provided tmpdir, which is expected to be // removed after the Load call. func (l *Library) Load(target, tmpdir string) (path string, err error) { - path, job, err := l.load(target, "", tmpdir) + job, err := l.load(target, "", tmpdir) if err != nil { return "", err } - if job != nil { - jobs := append([]*compileJob{job}, job.dependencies...) - err = runJobs(jobs) - } - return path, err + jobs := append([]*compileJob{job}, job.dependencies...) + err = runJobs(jobs) + return job.result, err } -// load returns a path to the library file for the given target, loading it from -// cache if possible. It will return a non-zero compiler job if the library -// wasn't cached, this job (and its dependencies) must be run before the library -// path is valid. +// load returns a compile job to build this library file for the given target +// and CPU. It may return a dummy compileJob if the library build is already +// cached. The path is stored as job.result but is only valid if the job and +// job.dependencies have been run. // The provided tmpdir will be used to store intermediary files and possibly the // output archive file, it is expected to be removed after use. -func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob, err error) { +func (l *Library) load(target, cpu, tmpdir string) (job *compileJob, err error) { // Try to load a precompiled library. precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, l.name+".a") if _, err := os.Stat(precompiledPath); err == nil { // Found a precompiled library for this OS/architecture. Return the path // directly. - return precompiledPath, nil, nil + return dummyCompileJob(precompiledPath), nil } var outfile string @@ -78,7 +76,7 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob // Try to fetch this library from the cache. if path, err := cacheLoad(outfile, l.sourcePaths(target)); path != "" || err != nil { // Cache hit. - return path, nil, err + return dummyCompileJob(path), nil } // Cache miss, build it now. @@ -86,7 +84,7 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob dir := filepath.Join(tmpdir, "build-lib-"+l.name) err = os.Mkdir(dir, 0777) if err != nil { - return "", nil, err + return nil, err } // Precalculate the flags to the compiler invocation. @@ -113,7 +111,8 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob arpath := filepath.Join(dir, l.name+".a") job = &compileJob{ description: "ar " + l.name + ".a", - run: func() error { + result: arpath, + run: func(*compileJob) error { // Create an archive of all object files. err := makeArchive(arpath, objs) if err != nil { @@ -133,7 +132,7 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob objs = append(objs, objpath) job.dependencies = append(job.dependencies, &compileJob{ description: "compile " + srcpath, - run: func() error { + run: func(*compileJob) error { var compileArgs []string compileArgs = append(compileArgs, args...) compileArgs = append(compileArgs, "-o", objpath, srcpath) @@ -146,5 +145,5 @@ func (l *Library) load(target, cpu, tmpdir string) (path string, job *compileJob }) } - return arpath, job, nil + return job, nil }