builder: parallelize most of the build
This commit parallelizes almost everything that can currently be parallelized. With that, it also introduces a framework for easily parallelizing other parts of the compiler. Code for baremetal targets already compiles slightly faster because it can parallelize the compilation of supporting assembly files. However, the speedup is especially noticeable when libraries (compiler-rt, picolibc) also need to be compiled: they will be compiled in parallel next to the Go files using all available cores. On my dual core laptop (4 cores if you count hyperthreading) this cuts compilation time roughly in half when compiling something for a Cortex-M board after running `tinygo clean`.
Этот коммит содержится в:
Ayke van Laethem
Ron Evans
родитель
3010466c55
коммит
d85ac4b3cc
5 изменённых файлов: 517 добавлений и 232 удалений
476
builder/build.go
476
builder/build.go
|
|
@ -65,10 +65,286 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
|
|||
return err
|
||||
}
|
||||
|
||||
// The slice of jobs that orchestrates most of the build.
|
||||
// This is somewhat like an in-memory Makefile with each job being a
|
||||
// Makefile target.
|
||||
var jobs []*compileJob
|
||||
|
||||
// Add job to compile and optimize all Go files at once.
|
||||
// TODO: parallelize this.
|
||||
var mod llvm.Module
|
||||
var stackSizeLoads []string
|
||||
programJob := &compileJob{
|
||||
description: "compile Go files",
|
||||
run: func() (err error) {
|
||||
mod, err = compileWholeProgram(pkgName, config, lprogram, machine)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Make sure stack sizes are loaded from a separate section so they can be
|
||||
// modified after linking.
|
||||
if config.AutomaticStackSize() {
|
||||
stackSizeLoads = transform.CreateStackSizeLoads(mod, config)
|
||||
}
|
||||
return
|
||||
},
|
||||
}
|
||||
jobs = append(jobs, programJob)
|
||||
|
||||
// Check whether we only need to create an object file.
|
||||
// If so, we don't need to link anything and will be finished quickly.
|
||||
outext := filepath.Ext(outpath)
|
||||
if outext == ".o" || outext == ".bc" || outext == ".ll" {
|
||||
// Run jobs to produce the LLVM module.
|
||||
err := runJobs(jobs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Generate output.
|
||||
switch outext {
|
||||
case ".o":
|
||||
llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ioutil.WriteFile(outpath, llvmBuf.Bytes(), 0666)
|
||||
case ".bc":
|
||||
data := llvm.WriteBitcodeToMemoryBuffer(mod).Bytes()
|
||||
return ioutil.WriteFile(outpath, data, 0666)
|
||||
case ".ll":
|
||||
data := []byte(mod.String())
|
||||
return ioutil.WriteFile(outpath, data, 0666)
|
||||
default:
|
||||
panic("unreachable")
|
||||
}
|
||||
}
|
||||
|
||||
// Act as a compiler driver, as we need to produce a complete executable.
|
||||
// First add all jobs necessary to build this object file, then afterwards
|
||||
// run all jobs in parallel as far as possible.
|
||||
|
||||
// Create a temporary directory for intermediary files.
|
||||
dir, err := ioutil.TempDir("", "tinygo")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
// Add job to write the output object file.
|
||||
objfile := filepath.Join(dir, "main.o")
|
||||
outputObjectFileJob := &compileJob{
|
||||
description: "generate output file",
|
||||
dependencies: []*compileJob{programJob},
|
||||
run: func() error {
|
||||
llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ioutil.WriteFile(objfile, llvmBuf.Bytes(), 0666)
|
||||
},
|
||||
}
|
||||
jobs = append(jobs, outputObjectFileJob)
|
||||
|
||||
// Prepare link command.
|
||||
linkerDependencies := []*compileJob{outputObjectFileJob}
|
||||
executable := filepath.Join(dir, "main")
|
||||
tmppath := executable // final file
|
||||
ldflags := append(config.LDFlags(), "-o", executable, objfile)
|
||||
|
||||
// Add compiler-rt dependency if needed. Usually this is a simple load from
|
||||
// a cache.
|
||||
if config.Target.RTLib == "compiler-rt" {
|
||||
path, job, err := CompilerRT.load(config.Triple(), dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if job != nil {
|
||||
// The library was not loaded from cache so needs to be compiled
|
||||
// (and then stored in the cache).
|
||||
jobs = append(jobs, job.dependencies...)
|
||||
jobs = append(jobs, job)
|
||||
linkerDependencies = append(linkerDependencies, job)
|
||||
}
|
||||
ldflags = append(ldflags, path)
|
||||
}
|
||||
|
||||
// Add libc dependency if needed.
|
||||
if config.Target.Libc == "picolibc" {
|
||||
path, job, err := Picolibc.load(config.Triple(), dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if job != nil {
|
||||
// The library needs to be compiled (cache miss).
|
||||
jobs = append(jobs, job.dependencies...)
|
||||
jobs = append(jobs, job)
|
||||
linkerDependencies = append(linkerDependencies, job)
|
||||
}
|
||||
ldflags = append(ldflags, path)
|
||||
}
|
||||
|
||||
// Add jobs to compile extra files. These files are in C or assembly and
|
||||
// contain things like the interrupt vector table and low level operations
|
||||
// such as stack switching.
|
||||
root := goenv.Get("TINYGOROOT")
|
||||
for i, path := range config.ExtraFiles() {
|
||||
abspath := filepath.Join(root, path)
|
||||
outpath := filepath.Join(dir, "extra-"+strconv.Itoa(i)+"-"+filepath.Base(path)+".o")
|
||||
job := &compileJob{
|
||||
description: "compile extra file " + path,
|
||||
run: func() error {
|
||||
err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, abspath)...)
|
||||
if err != nil {
|
||||
return &commandError{"failed to build", path, err}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
linkerDependencies = append(linkerDependencies, job)
|
||||
ldflags = append(ldflags, outpath)
|
||||
}
|
||||
|
||||
// Add jobs to compile C files in all packages. This is part of CGo.
|
||||
// TODO: do this as part of building the package to be able to link the
|
||||
// bitcode files together.
|
||||
for i, pkg := range lprogram.Sorted() {
|
||||
for j, filename := range pkg.CFiles {
|
||||
file := filepath.Join(pkg.Dir, filename)
|
||||
outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"."+strconv.Itoa(j)+"-"+filepath.Base(file)+".o")
|
||||
job := &compileJob{
|
||||
description: "compile CGo file " + file,
|
||||
run: func() error {
|
||||
err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, file)...)
|
||||
if err != nil {
|
||||
return &commandError{"failed to build", file, err}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
linkerDependencies = append(linkerDependencies, job)
|
||||
ldflags = append(ldflags, outpath)
|
||||
}
|
||||
}
|
||||
|
||||
// Linker flags from CGo lines:
|
||||
// #cgo LDFLAGS: foo
|
||||
if len(lprogram.LDFlags) > 0 {
|
||||
ldflags = append(ldflags, lprogram.LDFlags...)
|
||||
}
|
||||
|
||||
// Create a linker job, which links all object files together and does some
|
||||
// extra stuff that can only be done after linking.
|
||||
jobs = append(jobs, &compileJob{
|
||||
description: "link",
|
||||
dependencies: linkerDependencies,
|
||||
run: func() error {
|
||||
err = link(config.Target.Linker, ldflags...)
|
||||
if err != nil {
|
||||
return &commandError{"failed to link", executable, err}
|
||||
}
|
||||
|
||||
var calculatedStacks []string
|
||||
var stackSizes map[string]functionStackSize
|
||||
if config.Options.PrintStacks || config.AutomaticStackSize() {
|
||||
// Try to determine stack sizes at compile time.
|
||||
// Don't do this by default as it usually doesn't work on
|
||||
// unsupported architectures.
|
||||
calculatedStacks, stackSizes, err = determineStackSizes(mod, executable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if config.AutomaticStackSize() {
|
||||
// Modify the .tinygo_stacksizes section that contains a stack size
|
||||
// for each goroutine.
|
||||
err = modifyStackSizes(executable, stackSizeLoads, stackSizes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not modify stack sizes: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if config.Options.PrintSizes == "short" || config.Options.PrintSizes == "full" {
|
||||
sizes, err := loadProgramSize(executable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if config.Options.PrintSizes == "short" {
|
||||
fmt.Printf(" code data bss | flash ram\n")
|
||||
fmt.Printf("%7d %7d %7d | %7d %7d\n", sizes.Code, sizes.Data, sizes.BSS, sizes.Code+sizes.Data, sizes.Data+sizes.BSS)
|
||||
} else {
|
||||
fmt.Printf(" code rodata data bss | flash ram | package\n")
|
||||
for _, name := range sizes.sortedPackageNames() {
|
||||
pkgSize := sizes.Packages[name]
|
||||
fmt.Printf("%7d %7d %7d %7d | %7d %7d | %s\n", pkgSize.Code, pkgSize.ROData, pkgSize.Data, pkgSize.BSS, pkgSize.Flash(), pkgSize.RAM(), name)
|
||||
}
|
||||
fmt.Printf("%7d %7d %7d %7d | %7d %7d | (sum)\n", sizes.Sum.Code, sizes.Sum.ROData, sizes.Sum.Data, sizes.Sum.BSS, sizes.Sum.Flash(), sizes.Sum.RAM())
|
||||
fmt.Printf("%7d - %7d %7d | %7d %7d | (all)\n", sizes.Code, sizes.Data, sizes.BSS, sizes.Code+sizes.Data, sizes.Data+sizes.BSS)
|
||||
}
|
||||
}
|
||||
|
||||
// Print goroutine stack sizes, as far as possible.
|
||||
if config.Options.PrintStacks {
|
||||
printStacks(calculatedStacks, stackSizes)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
// Run all jobs to compile and link the program.
|
||||
// Do this now (instead of after elf-to-hex and similar conversions) as it
|
||||
// is simpler and cannot be parallelized.
|
||||
err = runJobs(jobs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get an Intel .hex file or .bin file from the .elf file.
|
||||
outputBinaryFormat := config.BinaryFormat(outext)
|
||||
switch outputBinaryFormat {
|
||||
case "elf":
|
||||
// do nothing, file is already in ELF format
|
||||
case "hex", "bin":
|
||||
// Extract raw binary, either encoding it as a hex file or as a raw
|
||||
// firmware file.
|
||||
tmppath = filepath.Join(dir, "main"+outext)
|
||||
err := objcopy(executable, tmppath, outputBinaryFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case "uf2":
|
||||
// Get UF2 from the .elf file.
|
||||
tmppath = filepath.Join(dir, "main"+outext)
|
||||
err := convertELFFileToUF2File(executable, tmppath, config.Target.UF2FamilyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case "esp32", "esp8266":
|
||||
// Special format for the ESP family of chips (parsed by the ROM
|
||||
// bootloader).
|
||||
tmppath = filepath.Join(dir, "main"+outext)
|
||||
err := makeESPFirmareImage(executable, tmppath, outputBinaryFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown output binary format: %s", outputBinaryFormat)
|
||||
}
|
||||
return action(BuildResult{
|
||||
Binary: tmppath,
|
||||
MainDir: lprogram.MainPkg().Dir,
|
||||
})
|
||||
}
|
||||
|
||||
// compileWholeProgram compiles the entire *loader.Program to a LLVM module and
|
||||
// applies most necessary optimizations and transformations.
|
||||
func compileWholeProgram(pkgName string, config *compileopts.Config, lprogram *loader.Program, machine llvm.TargetMachine) (llvm.Module, error) {
|
||||
// Compile AST to IR.
|
||||
mod, errs := compiler.CompileProgram(pkgName, lprogram, machine, config)
|
||||
if errs != nil {
|
||||
return newMultiError(errs)
|
||||
return mod, newMultiError(errs)
|
||||
}
|
||||
|
||||
if config.Options.PrintIR {
|
||||
|
|
@ -76,15 +352,15 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
|
|||
fmt.Println(mod.String())
|
||||
}
|
||||
if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
|
||||
return errors.New("verification error after IR construction")
|
||||
return mod, errors.New("verification error after IR construction")
|
||||
}
|
||||
|
||||
err = interp.Run(mod, config.DumpSSA())
|
||||
err := interp.Run(mod, config.DumpSSA())
|
||||
if err != nil {
|
||||
return err
|
||||
return mod, err
|
||||
}
|
||||
if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
|
||||
return errors.New("verification error after interpreting runtime.initAll")
|
||||
return mod, errors.New("verification error after interpreting runtime.initAll")
|
||||
}
|
||||
|
||||
if config.GOOS() != "darwin" {
|
||||
|
|
@ -99,7 +375,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
|
|||
if config.WasmAbi() == "js" {
|
||||
err := transform.ExternalInt64AsPtr(mod)
|
||||
if err != nil {
|
||||
return err
|
||||
return mod, err
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -128,10 +404,10 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
|
|||
errs = []error{errors.New("unknown optimization level: -opt=" + config.Options.Opt)}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return newMultiError(errs)
|
||||
return mod, newMultiError(errs)
|
||||
}
|
||||
if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {
|
||||
return errors.New("verification failure after LLVM optimization passes")
|
||||
return mod, errors.New("verification failure after LLVM optimization passes")
|
||||
}
|
||||
|
||||
// LLVM 11 by default tries to emit tail calls (even with the target feature
|
||||
|
|
@ -145,189 +421,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
|
|||
transform.DisableTailCalls(mod)
|
||||
}
|
||||
|
||||
// Make sure stack sizes are loaded from a separate section so they can be
|
||||
// modified after linking.
|
||||
var stackSizeLoads []string
|
||||
if config.AutomaticStackSize() {
|
||||
stackSizeLoads = transform.CreateStackSizeLoads(mod, config)
|
||||
}
|
||||
|
||||
// Generate output.
|
||||
outext := filepath.Ext(outpath)
|
||||
switch outext {
|
||||
case ".o":
|
||||
llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ioutil.WriteFile(outpath, llvmBuf.Bytes(), 0666)
|
||||
case ".bc":
|
||||
data := llvm.WriteBitcodeToMemoryBuffer(mod).Bytes()
|
||||
return ioutil.WriteFile(outpath, data, 0666)
|
||||
case ".ll":
|
||||
data := []byte(mod.String())
|
||||
return ioutil.WriteFile(outpath, data, 0666)
|
||||
default:
|
||||
// Act as a compiler driver.
|
||||
|
||||
// Create a temporary directory for intermediary files.
|
||||
dir, err := ioutil.TempDir("", "tinygo")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
// Write the object file.
|
||||
objfile := filepath.Join(dir, "main.o")
|
||||
llvmBuf, err := machine.EmitToMemoryBuffer(mod, llvm.ObjectFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ioutil.WriteFile(objfile, llvmBuf.Bytes(), 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare link command.
|
||||
executable := filepath.Join(dir, "main")
|
||||
tmppath := executable // final file
|
||||
ldflags := append(config.LDFlags(), "-o", executable, objfile)
|
||||
|
||||
// Load builtins library from the cache, possibly compiling it on the
|
||||
// fly.
|
||||
if config.Target.RTLib == "compiler-rt" {
|
||||
librt, err := CompilerRT.Load(config.Triple())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ldflags = append(ldflags, librt)
|
||||
}
|
||||
|
||||
// Add libc.
|
||||
if config.Target.Libc == "picolibc" {
|
||||
libc, err := Picolibc.Load(config.Triple())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ldflags = append(ldflags, libc)
|
||||
}
|
||||
|
||||
// Compile extra files.
|
||||
root := goenv.Get("TINYGOROOT")
|
||||
for i, path := range config.ExtraFiles() {
|
||||
abspath := filepath.Join(root, path)
|
||||
outpath := filepath.Join(dir, "extra-"+strconv.Itoa(i)+"-"+filepath.Base(path)+".o")
|
||||
err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, abspath)...)
|
||||
if err != nil {
|
||||
return &commandError{"failed to build", path, err}
|
||||
}
|
||||
ldflags = append(ldflags, outpath)
|
||||
}
|
||||
|
||||
// Compile C files in packages.
|
||||
// Gather the list of (C) file paths that should be included in the build.
|
||||
for i, pkg := range lprogram.Sorted() {
|
||||
for j, filename := range pkg.CFiles {
|
||||
file := filepath.Join(pkg.Dir, filename)
|
||||
outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"."+strconv.Itoa(j)+"-"+filepath.Base(file)+".o")
|
||||
err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, file)...)
|
||||
if err != nil {
|
||||
return &commandError{"failed to build", file, err}
|
||||
}
|
||||
ldflags = append(ldflags, outpath)
|
||||
}
|
||||
}
|
||||
|
||||
if len(lprogram.LDFlags) > 0 {
|
||||
ldflags = append(ldflags, lprogram.LDFlags...)
|
||||
}
|
||||
|
||||
// Link the object files together.
|
||||
err = link(config.Target.Linker, ldflags...)
|
||||
if err != nil {
|
||||
return &commandError{"failed to link", executable, err}
|
||||
}
|
||||
|
||||
var calculatedStacks []string
|
||||
var stackSizes map[string]functionStackSize
|
||||
if config.Options.PrintStacks || config.AutomaticStackSize() {
|
||||
// Try to determine stack sizes at compile time.
|
||||
// Don't do this by default as it usually doesn't work on
|
||||
// unsupported architectures.
|
||||
calculatedStacks, stackSizes, err = determineStackSizes(mod, executable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if config.AutomaticStackSize() {
|
||||
// Modify the .tinygo_stacksizes section that contains a stack size
|
||||
// for each goroutine.
|
||||
err = modifyStackSizes(executable, stackSizeLoads, stackSizes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not modify stack sizes: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if config.Options.PrintSizes == "short" || config.Options.PrintSizes == "full" {
|
||||
sizes, err := loadProgramSize(executable)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if config.Options.PrintSizes == "short" {
|
||||
fmt.Printf(" code data bss | flash ram\n")
|
||||
fmt.Printf("%7d %7d %7d | %7d %7d\n", sizes.Code, sizes.Data, sizes.BSS, sizes.Code+sizes.Data, sizes.Data+sizes.BSS)
|
||||
} else {
|
||||
fmt.Printf(" code rodata data bss | flash ram | package\n")
|
||||
for _, name := range sizes.sortedPackageNames() {
|
||||
pkgSize := sizes.Packages[name]
|
||||
fmt.Printf("%7d %7d %7d %7d | %7d %7d | %s\n", pkgSize.Code, pkgSize.ROData, pkgSize.Data, pkgSize.BSS, pkgSize.Flash(), pkgSize.RAM(), name)
|
||||
}
|
||||
fmt.Printf("%7d %7d %7d %7d | %7d %7d | (sum)\n", sizes.Sum.Code, sizes.Sum.ROData, sizes.Sum.Data, sizes.Sum.BSS, sizes.Sum.Flash(), sizes.Sum.RAM())
|
||||
fmt.Printf("%7d - %7d %7d | %7d %7d | (all)\n", sizes.Code, sizes.Data, sizes.BSS, sizes.Code+sizes.Data, sizes.Data+sizes.BSS)
|
||||
}
|
||||
}
|
||||
|
||||
// Print goroutine stack sizes, as far as possible.
|
||||
if config.Options.PrintStacks {
|
||||
printStacks(calculatedStacks, stackSizes)
|
||||
}
|
||||
|
||||
// Get an Intel .hex file or .bin file from the .elf file.
|
||||
outputBinaryFormat := config.BinaryFormat(outext)
|
||||
switch outputBinaryFormat {
|
||||
case "elf":
|
||||
// do nothing, file is already in ELF format
|
||||
case "hex", "bin":
|
||||
// Extract raw binary, either encoding it as a hex file or as a raw
|
||||
// firmware file.
|
||||
tmppath = filepath.Join(dir, "main"+outext)
|
||||
err := objcopy(executable, tmppath, outputBinaryFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case "uf2":
|
||||
// Get UF2 from the .elf file.
|
||||
tmppath = filepath.Join(dir, "main"+outext)
|
||||
err := convertELFFileToUF2File(executable, tmppath, config.Target.UF2FamilyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case "esp32", "esp8266":
|
||||
// Special format for the ESP family of chips (parsed by the ROM
|
||||
// bootloader).
|
||||
tmppath = filepath.Join(dir, "main"+outext)
|
||||
err := makeESPFirmareImage(executable, tmppath, outputBinaryFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown output binary format: %s", outputBinaryFormat)
|
||||
}
|
||||
return action(BuildResult{
|
||||
Binary: tmppath,
|
||||
MainDir: lprogram.MainPkg().Dir,
|
||||
})
|
||||
}
|
||||
return mod, nil
|
||||
}
|
||||
|
||||
// functionStackSizes keeps stack size information about a single function
|
||||
|
|
|
|||
|
|
@ -74,24 +74,16 @@ func cacheStore(tmppath, name, configKey string, sourceFiles []string) (string,
|
|||
return "", err
|
||||
}
|
||||
cachepath := filepath.Join(dir, name)
|
||||
err = moveFile(tmppath, cachepath)
|
||||
err = copyFile(tmppath, cachepath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return cachepath, nil
|
||||
}
|
||||
|
||||
// moveFile renames the file from src to dst. If renaming doesn't work (for
|
||||
// example, the rename crosses a filesystem boundary), the file is copied and
|
||||
// the old file is removed.
|
||||
func moveFile(src, dst string) error {
|
||||
err := os.Rename(src, dst)
|
||||
if err == nil {
|
||||
// Success!
|
||||
return nil
|
||||
}
|
||||
// Failed to move, probably a different filesystem.
|
||||
// Do a copy + remove.
|
||||
// copyFile copies the given file from src to dst. It can copy over
|
||||
// a possibly already existing file at the destination.
|
||||
func copyFile(src, dst string) error {
|
||||
inf, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
154
builder/jobs.go
Обычный файл
154
builder/jobs.go
Обычный файл
|
|
@ -0,0 +1,154 @@
|
|||
package builder
|
||||
|
||||
// This file implements a job runner for the compiler, which runs jobs in
|
||||
// parallel while taking care of dependencies.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Set to true to enable logging in the job runner. This may help to debug
|
||||
// concurrency or performance issues.
|
||||
const jobRunnerDebug = false
|
||||
|
||||
type jobState uint8
|
||||
|
||||
const (
|
||||
jobStateQueued jobState = iota // not yet running
|
||||
jobStateRunning // running
|
||||
jobStateFinished // finished running
|
||||
)
|
||||
|
||||
// compileJob is a single compiler job, comparable to a single Makefile target.
|
||||
// It is used to orchestrate various compiler tasks that can be run in parallel
|
||||
// but that have dependencies and thus have limitations in how they can be run.
|
||||
type compileJob struct {
|
||||
description string // description, only used for logging
|
||||
dependencies []*compileJob
|
||||
run func() error
|
||||
state jobState
|
||||
err error // error if finished
|
||||
duration time.Duration // how long it took to run this job (only set after finishing)
|
||||
}
|
||||
|
||||
// readyToRun returns whether this job is ready to run: it is itself not yet
|
||||
// started and all dependencies are finished.
|
||||
func (job *compileJob) readyToRun() bool {
|
||||
if job.state != jobStateQueued {
|
||||
// Already running or finished, so shouldn't be run again.
|
||||
return false
|
||||
}
|
||||
|
||||
// Check dependencies.
|
||||
for _, dep := range job.dependencies {
|
||||
if dep.state != jobStateFinished {
|
||||
// A dependency is not finished, so this job has to wait until it
|
||||
// is.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// All conditions are satisfied.
|
||||
return true
|
||||
}
|
||||
|
||||
// runJobs runs all the jobs indicated in the jobs slice and returns the error
|
||||
// of the first job that fails to run.
|
||||
// It runs all jobs in the order of the slice, as long as all dependencies have
|
||||
// already run. Therefore, if some jobs are preferred to run before others, they
|
||||
// should be ordered as such in this slice.
|
||||
func runJobs(jobs []*compileJob) error {
|
||||
// Create channels to communicate with the workers.
|
||||
doneChan := make(chan *compileJob)
|
||||
workerChan := make(chan *compileJob)
|
||||
defer close(workerChan)
|
||||
|
||||
// Start a number of workers.
|
||||
for i := 0; i < runtime.NumCPU(); i++ {
|
||||
if jobRunnerDebug {
|
||||
fmt.Println("## starting worker", i)
|
||||
}
|
||||
go jobWorker(workerChan, doneChan)
|
||||
}
|
||||
|
||||
// Send each job in the jobs slice to a worker, taking care of job
|
||||
// dependencies.
|
||||
numRunningJobs := 0
|
||||
var totalTime time.Duration
|
||||
start := time.Now()
|
||||
for {
|
||||
// If there are free workers, try starting a new job (if one is
|
||||
// available). If it succeeds, try again to fill the entire worker pool.
|
||||
if numRunningJobs < runtime.NumCPU() {
|
||||
jobToRun := nextJob(jobs)
|
||||
if jobToRun != nil {
|
||||
// Start job.
|
||||
if jobRunnerDebug {
|
||||
fmt.Println("## start: ", jobToRun.description)
|
||||
}
|
||||
jobToRun.state = jobStateRunning
|
||||
workerChan <- jobToRun
|
||||
numRunningJobs++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// When there are no jobs running, all jobs in the jobs slice must have
|
||||
// been finished. Therefore, the work is done.
|
||||
if numRunningJobs == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Wait until a job is finished.
|
||||
job := <-doneChan
|
||||
job.state = jobStateFinished
|
||||
numRunningJobs--
|
||||
totalTime += job.duration
|
||||
if jobRunnerDebug {
|
||||
fmt.Println("## finished:", job.description, "(time "+job.duration.String()+")")
|
||||
}
|
||||
if job.err != nil {
|
||||
return job.err
|
||||
}
|
||||
}
|
||||
|
||||
// Some statistics, if debugging.
|
||||
if jobRunnerDebug {
|
||||
// Total duration of running all jobs.
|
||||
duration := time.Since(start)
|
||||
fmt.Println("## total: ", duration)
|
||||
|
||||
// The individual time of each job combined. On a multicore system, this
|
||||
// should be lower than the total above.
|
||||
fmt.Println("## job sum: ", totalTime)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// nextJob returns the first ready-to-run job.
|
||||
// This is an implementation detail of runJobs.
|
||||
func nextJob(jobs []*compileJob) *compileJob {
|
||||
for _, job := range jobs {
|
||||
if job.readyToRun() {
|
||||
return job
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jobWorker is the goroutine that runs received jobs.
|
||||
// This is an implementation detail of runJobs.
|
||||
func jobWorker(workerChan, doneChan chan *compileJob) {
|
||||
for job := range workerChan {
|
||||
start := time.Now()
|
||||
err := job.run()
|
||||
if err != nil {
|
||||
job.err = err
|
||||
}
|
||||
job.duration = time.Since(start)
|
||||
doneChan <- job
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
package builder
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
|
@ -40,13 +39,33 @@ func (l *Library) sourcePaths(target string) []string {
|
|||
}
|
||||
|
||||
// Load the library archive, possibly generating and caching it if needed.
|
||||
func (l *Library) Load(target string) (path string, err error) {
|
||||
// The resulting file is stored in the provided tmpdir, which is expected to be
|
||||
// removed after the Load call.
|
||||
func (l *Library) Load(target, tmpdir string) (path string, err error) {
|
||||
path, job, err := l.load(target, tmpdir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if job != nil {
|
||||
jobs := append([]*compileJob{job}, job.dependencies...)
|
||||
err = runJobs(jobs)
|
||||
}
|
||||
return path, err
|
||||
}
|
||||
|
||||
// load returns a path to the library file for the given target, loading it from
|
||||
// cache if possible. It will return a non-zero compiler job if the library
|
||||
// wasn't cached, this job (and its dependencies) must be run before the library
|
||||
// path is valid.
|
||||
// The provided tmpdir will be used to store intermediary files and possibly the
|
||||
// output archive file, it is expected to be removed after use.
|
||||
func (l *Library) load(target, tmpdir string) (path string, job *compileJob, err error) {
|
||||
// Try to load a precompiled library.
|
||||
precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, l.name+".a")
|
||||
if _, err := os.Stat(precompiledPath); err == nil {
|
||||
// Found a precompiled library for this OS/architecture. Return the path
|
||||
// directly.
|
||||
return precompiledPath, nil
|
||||
return precompiledPath, nil, nil
|
||||
}
|
||||
|
||||
outfile := l.name + "-" + target + ".a"
|
||||
|
|
@ -54,19 +73,21 @@ func (l *Library) Load(target string) (path string, err error) {
|
|||
// Try to fetch this library from the cache.
|
||||
if path, err := cacheLoad(outfile, commands["clang"][0], l.sourcePaths(target)); path != "" || err != nil {
|
||||
// Cache hit.
|
||||
return path, err
|
||||
return path, nil, err
|
||||
}
|
||||
// Cache miss, build it now.
|
||||
|
||||
dirPrefix := "tinygo-" + l.name
|
||||
remapDir := filepath.Join(os.TempDir(), dirPrefix)
|
||||
dir, err := ioutil.TempDir(os.TempDir(), dirPrefix)
|
||||
remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name)
|
||||
dir := filepath.Join(tmpdir, "build-lib-"+l.name)
|
||||
err = os.Mkdir(dir, 0777)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", nil, err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
// Precalculate the flags to the compiler invocation.
|
||||
// Note: -fdebug-prefix-map is necessary to make the output archive
|
||||
// reproducible. Otherwise the temporary directory is stored in the archive
|
||||
// itself, which varies each run.
|
||||
args := append(l.cflags(), "-c", "-Oz", "-g", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir)
|
||||
if strings.HasPrefix(target, "arm") || strings.HasPrefix(target, "thumb") {
|
||||
args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft")
|
||||
|
|
@ -78,28 +99,44 @@ func (l *Library) Load(target string) (path string, err error) {
|
|||
args = append(args, "-march=rv64gc", "-mabi=lp64")
|
||||
}
|
||||
|
||||
// Compile all sources.
|
||||
// Create job to put all the object files in a single archive. This archive
|
||||
// file is the (static) library file.
|
||||
var objs []string
|
||||
arpath := filepath.Join(dir, l.name+".a")
|
||||
job = &compileJob{
|
||||
description: "ar " + l.name + ".a",
|
||||
run: func() error {
|
||||
// Create an archive of all object files.
|
||||
err := makeArchive(arpath, objs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Store this archive in the cache.
|
||||
_, err = cacheStore(arpath, outfile, commands["clang"][0], l.sourcePaths(target))
|
||||
return err
|
||||
},
|
||||
}
|
||||
|
||||
// Create jobs to compile all sources. These jobs are depended upon by the
|
||||
// archive job above, so must be run first.
|
||||
for _, srcpath := range l.sourcePaths(target) {
|
||||
srcpath := srcpath // avoid concurrency issues by redefining inside the loop
|
||||
objpath := filepath.Join(dir, filepath.Base(srcpath)+".o")
|
||||
objs = append(objs, objpath)
|
||||
// Note: -fdebug-prefix-map is necessary to make the output archive
|
||||
// reproducible. Otherwise the temporary directory is stored in the
|
||||
// archive itself, which varies each run.
|
||||
err := runCCompiler("clang", append(args, "-o", objpath, srcpath)...)
|
||||
if err != nil {
|
||||
return "", &commandError{"failed to build", srcpath, err}
|
||||
}
|
||||
job.dependencies = append(job.dependencies, &compileJob{
|
||||
description: "compile " + srcpath,
|
||||
run: func() error {
|
||||
var compileArgs []string
|
||||
compileArgs = append(compileArgs, args...)
|
||||
compileArgs = append(compileArgs, "-o", objpath, srcpath)
|
||||
err := runCCompiler("clang", compileArgs...)
|
||||
if err != nil {
|
||||
return &commandError{"failed to build", srcpath, err}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Put all the object files in a single archive. This archive file will be
|
||||
// used to statically link this library.
|
||||
arpath := filepath.Join(dir, l.name+".a")
|
||||
err = makeArchive(arpath, objs)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Store this archive in the cache.
|
||||
return cacheStore(arpath, outfile, commands["clang"][0], l.sourcePaths(target))
|
||||
return arpath, job, nil
|
||||
}
|
||||
|
|
|
|||
12
main.go
12
main.go
|
|
@ -932,9 +932,17 @@ func main() {
|
|||
fmt.Fprintf(os.Stderr, "Unknown library: %s\n", name)
|
||||
os.Exit(1)
|
||||
}
|
||||
path, err := lib.Load(*target)
|
||||
tmpdir, err := ioutil.TempDir("", "tinygo*")
|
||||
if err != nil {
|
||||
handleCompilerError(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
path, err := lib.Load(*target, tmpdir)
|
||||
handleCompilerError(err)
|
||||
copyFile(path, outpath)
|
||||
err = copyFile(path, outpath)
|
||||
if err != nil {
|
||||
handleCompilerError(err)
|
||||
}
|
||||
case "flash", "gdb":
|
||||
pkgName := filepath.ToSlash(flag.Arg(0))
|
||||
if command == "flash" {
|
||||
|
|
|
|||
Загрузка…
Создание таблицы
Сослаться в новой задаче