From 33f76d1c2e914bd477bb19060670ae5c3608cf41 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Tue, 6 Apr 2021 16:06:12 +0200 Subject: [PATCH] main: implement -ldflags="-X ..." This commit implements replacing some global variables with a different value, if the global variable has no initializer. For example, if you have: package main var version string you can replace the value with -ldflags="-X main.version=0.2". Right now it only works for uninitialized globals. The Go tooling also supports initialized globals (var version = "") but that is a bit hard to combine with how initialized globals are currently implemented. The current implementation still allows caching package IR files while making sure the values don't end up in the build cache. This means compiling a program multiple times with different values will use the cached package each time, inserting the string value only late in the build process. Fixes #1045 --- builder/build.go | 138 +++++++++++++++++++++++++++++++++++------ compileopts/options.go | 1 + main.go | 56 ++++++++++++++++- main_test.go | 12 ++++ testdata/ldflags.go | 9 +++ testdata/ldflags.txt | 1 + 6 files changed, 196 insertions(+), 21 deletions(-) create mode 100644 testdata/ldflags.go create mode 100644 testdata/ldflags.txt diff --git a/builder/build.go b/builder/build.go index 31947461..f16916ce 100644 --- a/builder/build.go +++ b/builder/build.go @@ -52,16 +52,17 @@ type BuildResult struct { // key, avoiding the need for recompiling all dependencies when only the // implementation of an imported package changes. type packageAction struct { - ImportPath string - CompilerVersion int // compiler.Version - InterpVersion int // interp.Version - LLVMVersion string - Config *compiler.Config - CFlags []string - FileHashes map[string]string // hash of every file that's part of the package - Imports map[string]string // map from imported package to action ID hash - OptLevel int // LLVM optimization level (0-3) - SizeLevel int // LLVM optimization for size level (0-2) + ImportPath string + CompilerVersion int // compiler.Version + InterpVersion int // interp.Version + LLVMVersion string + Config *compiler.Config + CFlags []string + FileHashes map[string]string // hash of every file that's part of the package + Imports map[string]string // map from imported package to action ID hash + OptLevel int // LLVM optimization level (0-3) + SizeLevel int // LLVM optimization for size level (0-2) + UndefinedGlobals []string // globals that are left as external globals (no initializer) } // Build performs a single package to executable Go build. It takes in a package @@ -133,19 +134,26 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil for _, pkg := range lprogram.Sorted() { pkg := pkg // necessary to avoid a race condition + var undefinedGlobals []string + for name := range config.Options.GlobalValues[pkg.Pkg.Path()] { + undefinedGlobals = append(undefinedGlobals, name) + } + sort.Strings(undefinedGlobals) + // Create a cache key: a hash from the action ID below that contains all // the parameters for the build. actionID := packageAction{ - ImportPath: pkg.ImportPath, - CompilerVersion: compiler.Version, - InterpVersion: interp.Version, - LLVMVersion: llvm.Version, - Config: compilerConfig, - CFlags: pkg.CFlags, - FileHashes: make(map[string]string, len(pkg.FileHashes)), - Imports: make(map[string]string, len(pkg.Pkg.Imports())), - OptLevel: optLevel, - SizeLevel: sizeLevel, + ImportPath: pkg.ImportPath, + CompilerVersion: compiler.Version, + InterpVersion: interp.Version, + LLVMVersion: llvm.Version, + Config: compilerConfig, + CFlags: pkg.CFlags, + FileHashes: make(map[string]string, len(pkg.FileHashes)), + Imports: make(map[string]string, len(pkg.Pkg.Imports())), + OptLevel: optLevel, + SizeLevel: sizeLevel, + UndefinedGlobals: undefinedGlobals, } for filePath, hash := range pkg.FileHashes { actionID.FileHashes[filePath] = hex.EncodeToString(hash) @@ -196,6 +204,25 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil return errors.New("verification error after compiling package " + pkg.ImportPath) } + // Erase all globals that are part of the undefinedGlobals list. + // This list comes from the -ldflags="-X pkg.foo=val" option. + // Instead of setting the value directly in the AST (which would + // mean the value, which may be a secret, is stored in the build + // cache), the global itself is left external (undefined) and is + // only set at the end of the compilation. + for _, name := range undefinedGlobals { + globalName := pkg.Pkg.Path() + "." + name + global := mod.NamedGlobal(globalName) + if global.IsNil() { + return errors.New("global not found: " + globalName) + } + name := global.Name() + newGlobal := llvm.AddGlobal(mod, global.Type().ElementType(), name+".tmp") + global.ReplaceAllUsesWith(newGlobal) + global.EraseFromParentAsGlobal() + newGlobal.SetName(name) + } + // Try to interpret package initializers at compile time. // It may only be possible to do this partially, in which case // it is completed after all IR files are linked. @@ -640,6 +667,12 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error { transform.ApplyFunctionSections(mod) // -ffunction-sections } + // Insert values from -ldflags="-X ..." into the IR. + err = setGlobalValues(mod, config.Options.GlobalValues) + if err != nil { + return err + } + // Browsers cannot handle external functions that have type i64 because it // cannot be represented exactly in JavaScript (JS only has doubles). To // keep functions interoperable, pass int64 types as pointers to @@ -677,6 +710,71 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error { return nil } +// setGlobalValues sets the global values from the -ldflags="-X ..." compiler +// option in the given module. An error may be returned if the global is not of +// the expected type. +func setGlobalValues(mod llvm.Module, globals map[string]map[string]string) error { + var pkgPaths []string + for pkgPath := range globals { + pkgPaths = append(pkgPaths, pkgPath) + } + sort.Strings(pkgPaths) + for _, pkgPath := range pkgPaths { + pkg := globals[pkgPath] + var names []string + for name := range pkg { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + value := pkg[name] + globalName := pkgPath + "." + name + global := mod.NamedGlobal(globalName) + if global.IsNil() || !global.Initializer().IsNil() { + // The global either does not exist (optimized away?) or has + // some value, in which case it has already been initialized at + // package init time. + continue + } + + // A strin is a {ptr, len} pair. We need these types to build the + // initializer. + initializerType := global.Type().ElementType() + if initializerType.TypeKind() != llvm.StructTypeKind || initializerType.StructName() == "" { + return fmt.Errorf("%s: not a string", globalName) + } + elementTypes := initializerType.StructElementTypes() + if len(elementTypes) != 2 { + return fmt.Errorf("%s: not a string", globalName) + } + + // Create a buffer for the string contents. + bufInitializer := mod.Context().ConstString(value, false) + buf := llvm.AddGlobal(mod, bufInitializer.Type(), ".string") + buf.SetInitializer(bufInitializer) + buf.SetAlignment(1) + buf.SetUnnamedAddr(true) + buf.SetLinkage(llvm.PrivateLinkage) + + // Create the string value, which is a {ptr, len} pair. + zero := llvm.ConstInt(mod.Context().Int32Type(), 0, false) + ptr := llvm.ConstGEP(buf, []llvm.Value{zero, zero}) + if ptr.Type() != elementTypes[0] { + return fmt.Errorf("%s: not a string", globalName) + } + length := llvm.ConstInt(elementTypes[1], uint64(len(value)), false) + initializer := llvm.ConstNamedStruct(initializerType, []llvm.Value{ + ptr, + length, + }) + + // Set the initializer. No initializer should be set at this point. + global.SetInitializer(initializer) + } + } + return nil +} + // functionStackSizes keeps stack size information about a single function // (usually a goroutine). type functionStackSize struct { diff --git a/compileopts/options.go b/compileopts/options.go index a12a92cd..7570fefe 100644 --- a/compileopts/options.go +++ b/compileopts/options.go @@ -30,6 +30,7 @@ type Options struct { PrintStacks bool Tags string WasmAbi string + GlobalValues map[string]map[string]string // map[pkgpath]map[varname]value TestConfig TestConfig Programmer string } diff --git a/main.go b/main.go index 0a8f0686..3d9bd6e7 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/google/shlex" "github.com/mattn/go-colorable" "github.com/tinygo-org/tinygo/builder" "github.com/tinygo-org/tinygo/compileopts" @@ -835,6 +836,52 @@ func handleCompilerError(err error) { } } +// This is a special type for the -X flag to parse the pkgpath.Var=stringVal +// format. It has to be a special type to allow multiple variables to be defined +// this way. +type globalValuesFlag map[string]map[string]string + +func (m globalValuesFlag) String() string { + return "pkgpath.Var=value" +} + +func (m globalValuesFlag) Set(value string) error { + equalsIndex := strings.IndexByte(value, '=') + if equalsIndex < 0 { + return errors.New("expected format pkgpath.Var=value") + } + pathAndName := value[:equalsIndex] + pointIndex := strings.LastIndexByte(pathAndName, '.') + if pointIndex < 0 { + return errors.New("expected format pkgpath.Var=value") + } + path := pathAndName[:pointIndex] + name := pathAndName[pointIndex+1:] + stringValue := value[equalsIndex+1:] + if m[path] == nil { + m[path] = make(map[string]string) + } + m[path][name] = stringValue + return nil +} + +// parseGoLinkFlag parses the -ldflags parameter. Its primary purpose right now +// is the -X flag, for setting the value of global string variables. +func parseGoLinkFlag(flagsString string) (map[string]map[string]string, error) { + set := flag.NewFlagSet("link", flag.ExitOnError) + globalVarValues := make(globalValuesFlag) + set.Var(globalVarValues, "X", "Set the value of the string variable to the given value.") + flags, err := shlex.Split(flagsString) + if err != nil { + return nil, err + } + err = set.Parse(flags) + if err != nil { + return nil, err + } + return map[string]map[string]string(globalVarValues), nil +} + func main() { if len(os.Args) < 2 { fmt.Fprintln(os.Stderr, "No command-line arguments supplied.") @@ -859,6 +906,7 @@ func main() { ocdOutput := flag.Bool("ocd-output", false, "print OCD daemon output during debug") port := flag.String("port", "", "flash port (can specify multiple candidates separated by commas)") programmer := flag.String("programmer", "", "which hardware programmer to use") + ldflags := flag.String("ldflags", "", "Go link tool compatible ldflags") wasmAbi := flag.String("wasm-abi", "", "WebAssembly ABI conventions: js (no i64 params) or generic") var flagJSON, flagDeps *bool @@ -888,6 +936,11 @@ func main() { } flag.CommandLine.Parse(os.Args[2:]) + globalVarValues, err := parseGoLinkFlag(*ldflags) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } options := &compileopts.Options{ Target: *target, Opt: *opt, @@ -902,13 +955,14 @@ func main() { PrintStacks: *printStacks, PrintCommands: *printCommands, Tags: *tags, + GlobalValues: globalVarValues, WasmAbi: *wasmAbi, Programmer: *programmer, } os.Setenv("CC", "clang -target="+*target) - err := options.Verify() + err = options.Verify() if err != nil { fmt.Fprintln(os.Stderr, err.Error()) usage() diff --git a/main_test.go b/main_test.go index 9f6d777c..69fe99e6 100644 --- a/main_test.go +++ b/main_test.go @@ -138,6 +138,18 @@ func TestCompiler(t *testing.T) { Opt: "0", }, nil) }) + + t.Run("ldflags", func(t *testing.T) { + t.Parallel() + runTestWithConfig("ldflags.go", "", t, &compileopts.Options{ + Opt: "z", + GlobalValues: map[string]map[string]string{ + "main": { + "someGlobal": "foobar", + }, + }, + }, nil) + }) }) } diff --git a/testdata/ldflags.go b/testdata/ldflags.go new file mode 100644 index 00000000..94db0dcb --- /dev/null +++ b/testdata/ldflags.go @@ -0,0 +1,9 @@ +package main + +// These globals can be changed using -ldflags="-X main.someGlobal=value". +// At the moment, only globals without an initializer can be replaced this way. +var someGlobal string + +func main() { + println("someGlobal:", someGlobal) +} diff --git a/testdata/ldflags.txt b/testdata/ldflags.txt new file mode 100644 index 00000000..0f39abf0 --- /dev/null +++ b/testdata/ldflags.txt @@ -0,0 +1 @@ +someGlobal: foobar