From 3e109fca5f289f5e97548f061d7e0e24c9727193 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 9 Dec 2021 19:37:13 +0100 Subject: [PATCH] builder: use build ID as cache key Instead of storing an increasing version number in relevant packages (compiler.Version, interp.Version, cgo.Version, ...), read the build ID from the currently running executable. This has several benefits: * All changes relevant to the compiled packages are caught. * No need to bump the version for each change to these packages. This avoids merge conflicts. * During development, `go install` is enough. No need to run `tinygo clean` all the time. Of course, the drawback is that it might be updated a bit more often than necessary but I think the overall benefit is big. Regular release users shouldn't see any difference. Because the tinygo binary stays the same, the cache works well. --- builder/build.go | 18 +++++---- builder/buildid.go | 92 ++++++++++++++++++++++++++++++++++++++++++++ cgo/cgo.go | 6 --- compiler/compiler.go | 5 --- interp/interp.go | 6 --- 5 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 builder/buildid.go diff --git a/builder/build.go b/builder/build.go index 84efa27a..46b2aae1 100644 --- a/builder/build.go +++ b/builder/build.go @@ -24,7 +24,6 @@ import ( "strings" "github.com/gofrs/flock" - "github.com/tinygo-org/tinygo/cgo" "github.com/tinygo-org/tinygo/compileopts" "github.com/tinygo-org/tinygo/compiler" "github.com/tinygo-org/tinygo/goenv" @@ -64,9 +63,8 @@ type BuildResult struct { // implementation of an imported package changes. type packageAction struct { ImportPath string - CGoVersion int // cgo.Version - CompilerVersion int // compiler.Version - InterpVersion int // interp.Version + CompilerBuildID string + TinyGoVersion string LLVMVersion string Config *compiler.Config CFlags []string @@ -84,6 +82,13 @@ type packageAction struct { // The error value may be of type *MultiError. Callers will likely want to check // for this case and print such errors individually. func Build(pkgName, outpath string, config *compileopts.Config, action func(BuildResult) error) error { + // Read the build ID of the tinygo binary. + // Used as a cache key for package builds. + compilerBuildID, err := ReadBuildID() + if err != nil { + return err + } + // Create a temporary directory for intermediary files. dir, err := ioutil.TempDir("", "tinygo") if err != nil { @@ -192,9 +197,8 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil // the parameters for the build. actionID := packageAction{ ImportPath: pkg.ImportPath, - CGoVersion: cgo.Version, - CompilerVersion: compiler.Version, - InterpVersion: interp.Version, + CompilerBuildID: string(compilerBuildID), + TinyGoVersion: goenv.Version, LLVMVersion: llvm.Version, Config: compilerConfig, CFlags: pkg.CFlags, diff --git a/builder/buildid.go b/builder/buildid.go new file mode 100644 index 00000000..e5529c5d --- /dev/null +++ b/builder/buildid.go @@ -0,0 +1,92 @@ +package builder + +import ( + "bytes" + "debug/elf" + "debug/macho" + "encoding/binary" + "fmt" + "io" + "os" + "runtime" +) + +// ReadBuildID reads the build ID from the currently running executable. +func ReadBuildID() ([]byte, error) { + executable, err := os.Executable() + if err != nil { + return nil, err + } + f, err := os.Open(executable) + if err != nil { + return nil, err + } + defer f.Close() + + switch runtime.GOOS { + case "linux", "freebsd": + // Read the GNU build id section. (Not sure about FreeBSD though...) + file, err := elf.NewFile(f) + if err != nil { + return nil, err + } + for _, section := range file.Sections { + if section.Type != elf.SHT_NOTE || section.Name != ".note.gnu.build-id" { + continue + } + buf := make([]byte, section.Size) + n, err := section.ReadAt(buf, 0) + if uint64(n) != section.Size || err != nil { + return nil, fmt.Errorf("could not read build id: %w", err) + } + return buf, nil + } + case "darwin": + // Read the LC_UUID load command, which contains the equivalent of a + // build ID. + file, err := macho.NewFile(f) + if err != nil { + return nil, err + } + for _, load := range file.Loads { + // Unfortunately, the debug/macho package doesn't support the + // LC_UUID command directly. So we have to read it from + // macho.LoadBytes. + load, ok := load.(macho.LoadBytes) + if !ok { + continue + } + raw := load.Raw() + command := binary.LittleEndian.Uint32(raw) + if command != 0x1b { + // Looking for the LC_UUID load command. + // LC_UUID is defined here as 0x1b: + // https://opensource.apple.com/source/xnu/xnu-4570.71.2/EXTERNAL_HEADERS/mach-o/loader.h.auto.html + continue + } + return raw[4:], nil + } + default: + // On other platforms (such as Windows) there isn't such a convenient + // build ID. Luckily, Go does have an equivalent of the build ID, which + // is stored as a special symbol named go.buildid. You can read it + // using `go tool buildid`, but the code below extracts it directly + // from the binary. + // Unfortunately, because of stripping with the -w flag, no symbol + // table might be available. Therefore, we have to scan the binary + // directly. Luckily the build ID is always at the start of the file. + // For details, see: + // https://github.com/golang/go/blob/master/src/cmd/internal/buildid/buildid.go + fileStart := make([]byte, 4096) + _, err := io.ReadFull(f, fileStart) + index := bytes.Index(fileStart, []byte("\xff Go build ID: \"")) + if index < 0 || index > len(fileStart)-103 { + return nil, fmt.Errorf("could not find build id in %s", err) + } + buf := fileStart[index : index+103] + if bytes.HasPrefix(buf, []byte("\xff Go build ID: \"")) && bytes.HasSuffix(buf, []byte("\"\n \xff")) { + return buf[len("\xff Go build ID: \"") : len(buf)-1], nil + } + } + return nil, fmt.Errorf("could not find build ID in %s", executable) +} diff --git a/cgo/cgo.go b/cgo/cgo.go index cd0c29d8..efaa949c 100644 --- a/cgo/cgo.go +++ b/cgo/cgo.go @@ -26,12 +26,6 @@ import ( "golang.org/x/tools/go/ast/astutil" ) -// Version of the cgo package. It must be incremented whenever the cgo package -// is changed in a way that affects the output so that cached package builds -// will be invalidated. -// This version is independent of the TinyGo version number. -const Version = 1 // last change: run libclang once per Go file - // cgoPackage holds all CGo-related information of a package. type cgoPackage struct { generated *ast.File diff --git a/compiler/compiler.go b/compiler/compiler.go index 638f6dc0..d5c1bce0 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -20,11 +20,6 @@ import ( "tinygo.org/x/go-llvm" ) -// Version of the compiler pacakge. Must be incremented each time the compiler -// package changes in a way that affects the generated LLVM module. -// This version is independent of the TinyGo version number. -const Version = 25 // last change: add "target-cpu" and "target-features" attributes - func init() { llvm.InitializeAllTargets() llvm.InitializeAllTargetMCs() diff --git a/interp/interp.go b/interp/interp.go index 78896179..3f5e9e50 100644 --- a/interp/interp.go +++ b/interp/interp.go @@ -11,12 +11,6 @@ import ( "tinygo.org/x/go-llvm" ) -// Version of the interp package. It must be incremented whenever the interp -// package is changed in a way that affects the output so that cached package -// builds will be invalidated. -// This version is independent of the TinyGo version number. -const Version = 2 // last change: fix GEP on untyped pointers - // Enable extra checks, which should be disabled by default. // This may help track down bugs by adding a few more sanity checks. const checks = true