main: implement -ldflags="-X ..."

This commit implements replacing some global variables with a different
value, if the global variable has no initializer. For example, if you
have:

    package main

    var version string

you can replace the value with -ldflags="-X main.version=0.2".

Right now it only works for uninitialized globals. The Go tooling also
supports initialized globals (var version = "<undefined>") but that is a
bit hard to combine with how initialized globals are currently
implemented.

The current implementation still allows caching package IR files while
making sure the values don't end up in the build cache. This means
compiling a program multiple times with different values will use the
cached package each time, inserting the string value only late in the
build process.

Fixes #1045

builder/build.go

@@ -62,6 +62,7 @@ type packageAction struct {
 	Imports          map[string]string // map from imported package to action ID hash
 	OptLevel         int               // LLVM optimization level (0-3)
 	SizeLevel        int               // LLVM optimization for size level (0-2)
+	UndefinedGlobals []string          // globals that are left as external globals (no initializer)
 }
 
 // Build performs a single package to executable Go build. It takes in a package
@@ -133,6 +134,12 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	for _, pkg := range lprogram.Sorted() {
 		pkg := pkg // necessary to avoid a race condition
 
+		var undefinedGlobals []string
+		for name := range config.Options.GlobalValues[pkg.Pkg.Path()] {
+			undefinedGlobals = append(undefinedGlobals, name)
+		}
+		sort.Strings(undefinedGlobals)
+
 		// Create a cache key: a hash from the action ID below that contains all
 		// the parameters for the build.
 		actionID := packageAction{
@@ -146,6 +153,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			Imports:          make(map[string]string, len(pkg.Pkg.Imports())),
 			OptLevel:         optLevel,
 			SizeLevel:        sizeLevel,
+			UndefinedGlobals: undefinedGlobals,
 		}
 		for filePath, hash := range pkg.FileHashes {
 			actionID.FileHashes[filePath] = hex.EncodeToString(hash)
@@ -196,6 +204,25 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 					return errors.New("verification error after compiling package " + pkg.ImportPath)
 				}
 
+				// Erase all globals that are part of the undefinedGlobals list.
+				// This list comes from the -ldflags="-X pkg.foo=val" option.
+				// Instead of setting the value directly in the AST (which would
+				// mean the value, which may be a secret, is stored in the build
+				// cache), the global itself is left external (undefined) and is
+				// only set at the end of the compilation.
+				for _, name := range undefinedGlobals {
+					globalName := pkg.Pkg.Path() + "." + name
+					global := mod.NamedGlobal(globalName)
+					if global.IsNil() {
+						return errors.New("global not found: " + globalName)
+					}
+					name := global.Name()
+					newGlobal := llvm.AddGlobal(mod, global.Type().ElementType(), name+".tmp")
+					global.ReplaceAllUsesWith(newGlobal)
+					global.EraseFromParentAsGlobal()
+					newGlobal.SetName(name)
+				}
+
 				// Try to interpret package initializers at compile time.
 				// It may only be possible to do this partially, in which case
 				// it is completed after all IR files are linked.
@@ -640,6 +667,12 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error {
 		transform.ApplyFunctionSections(mod) // -ffunction-sections
 	}
 
+	// Insert values from -ldflags="-X ..." into the IR.
+	err = setGlobalValues(mod, config.Options.GlobalValues)
+	if err != nil {
+		return err
+	}
+
 	// Browsers cannot handle external functions that have type i64 because it
 	// cannot be represented exactly in JavaScript (JS only has doubles). To
 	// keep functions interoperable, pass int64 types as pointers to
@@ -677,6 +710,71 @@ func optimizeProgram(mod llvm.Module, config *compileopts.Config) error {
 	return nil
 }
 
+// setGlobalValues sets the global values from the -ldflags="-X ..." compiler
+// option in the given module. An error may be returned if the global is not of
+// the expected type.
+func setGlobalValues(mod llvm.Module, globals map[string]map[string]string) error {
+	var pkgPaths []string
+	for pkgPath := range globals {
+		pkgPaths = append(pkgPaths, pkgPath)
+	}
+	sort.Strings(pkgPaths)
+	for _, pkgPath := range pkgPaths {
+		pkg := globals[pkgPath]
+		var names []string
+		for name := range pkg {
+			names = append(names, name)
+		}
+		sort.Strings(names)
+		for _, name := range names {
+			value := pkg[name]
+			globalName := pkgPath + "." + name
+			global := mod.NamedGlobal(globalName)
+			if global.IsNil() || !global.Initializer().IsNil() {
+				// The global either does not exist (optimized away?) or has
+				// some value, in which case it has already been initialized at
+				// package init time.
+				continue
+			}
+
+			// A strin is a {ptr, len} pair. We need these types to build the
+			// initializer.
+			initializerType := global.Type().ElementType()
+			if initializerType.TypeKind() != llvm.StructTypeKind || initializerType.StructName() == "" {
+				return fmt.Errorf("%s: not a string", globalName)
+			}
+			elementTypes := initializerType.StructElementTypes()
+			if len(elementTypes) != 2 {
+				return fmt.Errorf("%s: not a string", globalName)
+			}
+
+			// Create a buffer for the string contents.
+			bufInitializer := mod.Context().ConstString(value, false)
+			buf := llvm.AddGlobal(mod, bufInitializer.Type(), ".string")
+			buf.SetInitializer(bufInitializer)
+			buf.SetAlignment(1)
+			buf.SetUnnamedAddr(true)
+			buf.SetLinkage(llvm.PrivateLinkage)
+
+			// Create the string value, which is a {ptr, len} pair.
+			zero := llvm.ConstInt(mod.Context().Int32Type(), 0, false)
+			ptr := llvm.ConstGEP(buf, []llvm.Value{zero, zero})
+			if ptr.Type() != elementTypes[0] {
+				return fmt.Errorf("%s: not a string", globalName)
+			}
+			length := llvm.ConstInt(elementTypes[1], uint64(len(value)), false)
+			initializer := llvm.ConstNamedStruct(initializerType, []llvm.Value{
+				ptr,
+				length,
+			})
+
+			// Set the initializer. No initializer should be set at this point.
+			global.SetInitializer(initializer)
+		}
+	}
+	return nil
+}
+
 // functionStackSizes keeps stack size information about a single function
 // (usually a goroutine).
 type functionStackSize struct {

compileopts/options.go

@@ -30,6 +30,7 @@ type Options struct {
 	PrintStacks   bool
 	Tags          string
 	WasmAbi       string
+	GlobalValues  map[string]map[string]string // map[pkgpath]map[varname]value
 	TestConfig    TestConfig
 	Programmer    string
 }

main.go

@@ -18,6 +18,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/google/shlex"
 	"github.com/mattn/go-colorable"
 	"github.com/tinygo-org/tinygo/builder"
 	"github.com/tinygo-org/tinygo/compileopts"
@@ -835,6 +836,52 @@ func handleCompilerError(err error) {
 	}
 }
 
+// This is a special type for the -X flag to parse the pkgpath.Var=stringVal
+// format. It has to be a special type to allow multiple variables to be defined
+// this way.
+type globalValuesFlag map[string]map[string]string
+
+func (m globalValuesFlag) String() string {
+	return "pkgpath.Var=value"
+}
+
+func (m globalValuesFlag) Set(value string) error {
+	equalsIndex := strings.IndexByte(value, '=')
+	if equalsIndex < 0 {
+		return errors.New("expected format pkgpath.Var=value")
+	}
+	pathAndName := value[:equalsIndex]
+	pointIndex := strings.LastIndexByte(pathAndName, '.')
+	if pointIndex < 0 {
+		return errors.New("expected format pkgpath.Var=value")
+	}
+	path := pathAndName[:pointIndex]
+	name := pathAndName[pointIndex+1:]
+	stringValue := value[equalsIndex+1:]
+	if m[path] == nil {
+		m[path] = make(map[string]string)
+	}
+	m[path][name] = stringValue
+	return nil
+}
+
+// parseGoLinkFlag parses the -ldflags parameter. Its primary purpose right now
+// is the -X flag, for setting the value of global string variables.
+func parseGoLinkFlag(flagsString string) (map[string]map[string]string, error) {
+	set := flag.NewFlagSet("link", flag.ExitOnError)
+	globalVarValues := make(globalValuesFlag)
+	set.Var(globalVarValues, "X", "Set the value of the string variable to the given value.")
+	flags, err := shlex.Split(flagsString)
+	if err != nil {
+		return nil, err
+	}
+	err = set.Parse(flags)
+	if err != nil {
+		return nil, err
+	}
+	return map[string]map[string]string(globalVarValues), nil
+}
+
 func main() {
 	if len(os.Args) < 2 {
 		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
@@ -859,6 +906,7 @@ func main() {
 	ocdOutput := flag.Bool("ocd-output", false, "print OCD daemon output during debug")
 	port := flag.String("port", "", "flash port (can specify multiple candidates separated by commas)")
 	programmer := flag.String("programmer", "", "which hardware programmer to use")
+	ldflags := flag.String("ldflags", "", "Go link tool compatible ldflags")
 	wasmAbi := flag.String("wasm-abi", "", "WebAssembly ABI conventions: js (no i64 params) or generic")
 
 	var flagJSON, flagDeps *bool
@@ -888,6 +936,11 @@ func main() {
 	}
 
 	flag.CommandLine.Parse(os.Args[2:])
+	globalVarValues, err := parseGoLinkFlag(*ldflags)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
 	options := &compileopts.Options{
 		Target:        *target,
 		Opt:           *opt,
@@ -902,13 +955,14 @@ func main() {
 		PrintStacks:   *printStacks,
 		PrintCommands: *printCommands,
 		Tags:          *tags,
+		GlobalValues:  globalVarValues,
 		WasmAbi:       *wasmAbi,
 		Programmer:    *programmer,
 	}
 
 	os.Setenv("CC", "clang -target="+*target)
 
-	err := options.Verify()
+	err = options.Verify()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err.Error())
 		usage()

main_test.go

@@ -138,6 +138,18 @@ func TestCompiler(t *testing.T) {
 				Opt: "0",
 			}, nil)
 		})
+
+		t.Run("ldflags", func(t *testing.T) {
+			t.Parallel()
+			runTestWithConfig("ldflags.go", "", t, &compileopts.Options{
+				Opt: "z",
+				GlobalValues: map[string]map[string]string{
+					"main": {
+						"someGlobal": "foobar",
+					},
+				},
+			}, nil)
+		})
 	})
 }
 

testdata/ldflags.go

@@ -0,0 +1,9 @@
+package main
+
+// These globals can be changed using -ldflags="-X main.someGlobal=value".
+// At the moment, only globals without an initializer can be replaced this way.
+var someGlobal string
+
+func main() {
+	println("someGlobal:", someGlobal)
+}

testdata/ldflags.txt

@@ -0,0 +1 @@
+someGlobal: foobar