From 9dd249a431e7c64a897e7d256ffcfa1a7a7aed01 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 26 May 2022 14:38:34 +0200 Subject: [PATCH] builder: refactor package compile job This commit moves the calculation of the package action ID (cache key) into a separate job. At the moment, this won't have a big effect but this change is necessary for some future changes I want to make. --- builder/build.go | 99 ++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 42 deletions(-) diff --git a/builder/build.go b/builder/build.go index 62912139..d68f36a3 100644 --- a/builder/build.go +++ b/builder/build.go @@ -209,8 +209,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil // Add jobs to compile each package. // Packages that have a cache hit will not be compiled again. var packageJobs []*compileJob - packageBitcodePaths := make(map[string]string) - packageActionIDs := make(map[string]string) + packageActionIDJobs := make(map[string]*compileJob) if config.Options.GlobalValues["runtime"]["buildVersion"] == "" { version := goenv.Version @@ -235,52 +234,68 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil } sort.Strings(undefinedGlobals) - // Create a cache key: a hash from the action ID below that contains all - // the parameters for the build. - actionID := packageAction{ - ImportPath: pkg.ImportPath, - CompilerBuildID: string(compilerBuildID), - TinyGoVersion: goenv.Version, - LLVMVersion: llvm.Version, - Config: compilerConfig, - CFlags: pkg.CFlags, - FileHashes: make(map[string]string, len(pkg.FileHashes)), - Imports: make(map[string]string, len(pkg.Pkg.Imports())), - OptLevel: optLevel, - SizeLevel: sizeLevel, - UndefinedGlobals: undefinedGlobals, - } - for filePath, hash := range pkg.FileHashes { - actionID.FileHashes[filePath] = hex.EncodeToString(hash) - } + // Action ID jobs need to know the action ID of all the jobs the package + // imports. + var importedPackages []*compileJob for _, imported := range pkg.Pkg.Imports() { - hash, ok := packageActionIDs[imported.Path()] + job, ok := packageActionIDJobs[imported.Path()] if !ok { return fmt.Errorf("package %s imports %s but couldn't find dependency", pkg.ImportPath, imported.Path()) } - actionID.Imports[imported.Path()] = hash + importedPackages = append(importedPackages, job) } - buf, err := json.Marshal(actionID) - if err != nil { - panic(err) // shouldn't happen + + // Create a job that will calculate the action ID for a package compile + // job. The action ID is the cache key that is used for caching this + // package. + packageActionIDJob := &compileJob{ + description: "calculate cache key for package " + pkg.ImportPath, + dependencies: importedPackages, + run: func(job *compileJob) error { + // Create a cache key: a hash from the action ID below that contains all + // the parameters for the build. + actionID := packageAction{ + ImportPath: pkg.ImportPath, + CompilerBuildID: string(compilerBuildID), + TinyGoVersion: goenv.Version, + LLVMVersion: llvm.Version, + Config: compilerConfig, + CFlags: pkg.CFlags, + FileHashes: make(map[string]string, len(pkg.FileHashes)), + Imports: make(map[string]string, len(pkg.Pkg.Imports())), + OptLevel: optLevel, + SizeLevel: sizeLevel, + UndefinedGlobals: undefinedGlobals, + } + for filePath, hash := range pkg.FileHashes { + actionID.FileHashes[filePath] = hex.EncodeToString(hash) + } + for i, imported := range pkg.Pkg.Imports() { + actionID.Imports[imported.Path()] = importedPackages[i].result + } + buf, err := json.Marshal(actionID) + if err != nil { + return err // shouldn't happen + } + hash := sha512.Sum512_224(buf) + job.result = hex.EncodeToString(hash[:]) + return nil + }, } - hash := sha512.Sum512_224(buf) - packageActionIDs[pkg.ImportPath] = hex.EncodeToString(hash[:]) + packageActionIDJobs[pkg.ImportPath] = packageActionIDJob - // Determine the path of the bitcode file (which is a serialized version - // of a LLVM module). - bitcodePath := filepath.Join(cacheDir, "pkg-"+hex.EncodeToString(hash[:])+".bc") - packageBitcodePaths[pkg.ImportPath] = bitcodePath - - // The package has not yet been compiled, so create a job to do so. + // Now create the job to actually build the package. It will exit early + // if the package is already compiled. job := &compileJob{ - description: "compile package " + pkg.ImportPath, - run: func(*compileJob) error { + description: "compile package " + pkg.ImportPath, + dependencies: []*compileJob{packageActionIDJob}, + run: func(job *compileJob) error { + job.result = filepath.Join(cacheDir, "pkg-"+packageActionIDJob.result+".bc") // Acquire a lock (if supported). - unlock := lock(bitcodePath + ".lock") + unlock := lock(job.result + ".lock") defer unlock() - if _, err := os.Stat(bitcodePath); err == nil { + if _, err := os.Stat(job.result); err == nil { // Already cached, don't recreate this package. return nil } @@ -401,7 +416,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil // Write to a temporary path that is renamed to the destination // file to avoid race conditions with other TinyGo invocatiosn // that might also be compiling this package at the same time. - f, err := ioutil.TempFile(filepath.Dir(bitcodePath), filepath.Base(bitcodePath)) + f, err := ioutil.TempFile(filepath.Dir(job.result), filepath.Base(job.result)) if err != nil { return err } @@ -421,13 +436,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil if err != nil { // WriteBitcodeToFile doesn't produce a useful error on its // own, so create a somewhat useful error message here. - return fmt.Errorf("failed to write bitcode for package %s to file %s", pkg.ImportPath, bitcodePath) + return fmt.Errorf("failed to write bitcode for package %s to file %s", pkg.ImportPath, job.result) } err = f.Close() if err != nil { return err } - return os.Rename(f.Name(), bitcodePath) + return os.Rename(f.Name(), job.result) }, } packageJobs = append(packageJobs, job) @@ -451,8 +466,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil // anything, it only links the bitcode files together. ctx := llvm.NewContext() mod = ctx.NewModule("main") - for _, pkg := range lprogram.Sorted() { - pkgMod, err := ctx.ParseBitcodeFile(packageBitcodePaths[pkg.ImportPath]) + for _, pkgJob := range packageJobs { + pkgMod, err := ctx.ParseBitcodeFile(pkgJob.result) if err != nil { return fmt.Errorf("failed to load bitcode file: %w", err) }