builder: refactor clang include headers

Set -resource-dir in a central place instead of passing the header path
around everywhere and adding it using the `-I` flag. I believe this is
closer to how Clang is intended to be used.

This change was inspired by my attempt to add a Nix flake file to
TinyGo.

builder/build.go

@@ -211,7 +211,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
 	defer machine.Dispose()
 
 	// Load entire program AST into memory.
-	lprogram, err := loader.Load(config, pkgName, config.ClangHeaders, types.Config{
+	lprogram, err := loader.Load(config, pkgName, types.Config{
 		Sizes: compiler.Sizes(machine),
 	})
 	if err != nil {
@@ -662,7 +662,7 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
 		job := &compileJob{
 			description: "compile extra file " + path,
 			run: func(job *compileJob) error {
-				result, err := compileAndCacheCFile(abspath, tmpdir, config.CFlags(), config.Options.PrintCommands)
+				result, err := compileAndCacheCFile(abspath, tmpdir, config.CFlags(false), config.Options.PrintCommands)
 				job.result = result
 				return err
 			},

builder/builder_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/tinygo-org/tinygo/compileopts"
-	"github.com/tinygo-org/tinygo/goenv"
 	"tinygo.org/x/go-llvm"
 )
 
@@ -74,7 +73,6 @@ func TestClangAttributes(t *testing.T) {
 
 func testClangAttributes(t *testing.T, options *compileopts.Options) {
 	testDir := t.TempDir()
-	clangHeaderPath := getClangHeaderPath(goenv.Get("TINYGOROOT"))
 
 	ctx := llvm.NewContext()
 	defer ctx.Dispose()
@@ -84,9 +82,8 @@ func testClangAttributes(t *testing.T, options *compileopts.Options) {
 		t.Fatalf("could not load target: %s", err)
 	}
 	config := compileopts.Config{
-		Options:      options,
+		Options: options,
-		Target:       target,
+		Target:  target,
-		ClangHeaders: clangHeaderPath,
 	}
 
 	// Create a very simple C input file.
@@ -98,7 +95,7 @@ func testClangAttributes(t *testing.T, options *compileopts.Options) {
 
 	// Compile this file using Clang.
 	outpath := filepath.Join(testDir, "test.bc")
-	flags := append([]string{"-c", "-emit-llvm", "-o", outpath, srcpath}, config.CFlags()...)
+	flags := append([]string{"-c", "-emit-llvm", "-o", outpath, srcpath}, config.CFlags(false)...)
 	if config.GOOS() == "darwin" {
 		// Silence some warnings that happen when testing GOOS=darwin on
 		// something other than MacOS.

builder/config.go

@@ -33,13 +33,10 @@ func NewConfig(options *compileopts.Options) (*compileopts.Config, error) {
 		return nil, fmt.Errorf("requires go version 1.18 through 1.21, got go%d.%d", major, minor)
 	}
 
-	clangHeaderPath := getClangHeaderPath(goenv.Get("TINYGOROOT"))
-
 	return &compileopts.Config{
 		Options:        options,
 		Target:         spec,
 		GoMinorVersion: minor,
-		ClangHeaders:   clangHeaderPath,
 		TestConfig:     options.TestConfig,
 	}, nil
 }

builder/env.go

@@ -1,105 +0,0 @@
-package builder
-
-import (
-	"errors"
-	"io/fs"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	"tinygo.org/x/go-llvm"
-)
-
-// getClangHeaderPath returns the path to the built-in Clang headers. It tries
-// multiple locations, which should make it find the directory when installed in
-// various ways.
-func getClangHeaderPath(TINYGOROOT string) string {
-	// Check whether we're running from the source directory.
-	path := filepath.Join(TINYGOROOT, "llvm-project", "clang", "lib", "Headers")
-	if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
-		return path
-	}
-
-	// Check whether we're running from the installation directory.
-	path = filepath.Join(TINYGOROOT, "lib", "clang", "include")
-	if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
-		return path
-	}
-
-	// It looks like we are built with a system-installed LLVM. Do a last
-	// attempt: try to use Clang headers relative to the clang binary.
-	llvmMajor := strings.Split(llvm.Version, ".")[0]
-	for _, cmdName := range commands["clang"] {
-		binpath, err := exec.LookPath(cmdName)
-		if err == nil {
-			// This should be the command that will also be used by
-			// execCommand. To avoid inconsistencies, make sure we use the
-			// headers relative to this command.
-			binpath, err = filepath.EvalSymlinks(binpath)
-			if err != nil {
-				// Unexpected.
-				return ""
-			}
-			// Example executable:
-			//     /usr/lib/llvm-9/bin/clang
-			// Example include path:
-			//     /usr/lib/llvm-9/lib64/clang/9.0.1/include/
-			llvmRoot := filepath.Dir(filepath.Dir(binpath))
-			clangVersionRoot := filepath.Join(llvmRoot, "lib64", "clang")
-			dirs64, err64 := ioutil.ReadDir(clangVersionRoot)
-			// Example include path:
-			//     /usr/lib/llvm-9/lib/clang/9.0.1/include/
-			clangVersionRoot = filepath.Join(llvmRoot, "lib", "clang")
-			dirs32, err32 := ioutil.ReadDir(clangVersionRoot)
-			if err64 != nil && err32 != nil {
-				// Unexpected.
-				continue
-			}
-			dirnames := make([]string, len(dirs64)+len(dirs32))
-			dirCount := 0
-			for _, d := range dirs32 {
-				name := d.Name()
-				if name == llvmMajor || strings.HasPrefix(name, llvmMajor+".") {
-					dirnames[dirCount] = filepath.Join(llvmRoot, "lib", "clang", name)
-					dirCount++
-				}
-			}
-			for _, d := range dirs64 {
-				name := d.Name()
-				if name == llvmMajor || strings.HasPrefix(name, llvmMajor+".") {
-					dirnames[dirCount] = filepath.Join(llvmRoot, "lib64", "clang", name)
-					dirCount++
-				}
-			}
-			sort.Strings(dirnames)
-			// Check for the highest version first.
-			for i := dirCount - 1; i >= 0; i-- {
-				path := filepath.Join(dirnames[i], "include")
-				_, err := os.Stat(filepath.Join(path, "stdint.h"))
-				if err == nil {
-					return path
-				}
-			}
-		}
-	}
-
-	// On Arch Linux, the clang executable is stored in /usr/bin rather than being symlinked from there.
-	// Search directly in /usr/lib for clang.
-	if matches, err := filepath.Glob("/usr/lib/clang/" + llvmMajor + ".*.*"); err == nil {
-		// Check for the highest version first.
-		sort.Strings(matches)
-		for i := len(matches) - 1; i >= 0; i-- {
-			path := filepath.Join(matches[i], "include")
-			_, err := os.Stat(filepath.Join(path, "stdint.h"))
-			if err == nil {
-				return path
-			}
-		}
-	}
-
-	// Could not find it.
-	return ""
-}

builder/library.go

@@ -143,6 +143,10 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
 	// reproducible. Otherwise the temporary directory is stored in the archive
 	// itself, which varies each run.
 	args := append(l.cflags(target, headerPath), "-c", "-Oz", "-gdwarf-4", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir)
+	resourceDir := goenv.ClangResourceDir(false)
+	if resourceDir != "" {
+		args = append(args, "-resource-dir="+resourceDir)
+	}
 	cpu := config.CPU()
 	if cpu != "" {
 		// X86 has deprecated the -mcpu flag, so we need to use -march instead.

builder/tools.go

@@ -1,7 +1,6 @@
 package builder
 
 import (
-	"errors"
 	"os"
 	"os/exec"
 
@@ -12,11 +11,6 @@ import (
 func runCCompiler(flags ...string) error {
 	if hasBuiltinTools {
 		// Compile this with the internal Clang compiler.
-		headerPath := getClangHeaderPath(goenv.Get("TINYGOROOT"))
-		if headerPath == "" {
-			return errors.New("could not locate Clang headers")
-		}
-		flags = append(flags, "-I"+headerPath)
 		cmd := exec.Command(os.Args[0], append([]string{"clang"}, flags...)...)
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr

cgo/cgo.go

@@ -165,7 +165,7 @@ func GoBytes(ptr unsafe.Pointer, length C.int) []byte {
 // functions), the CFLAGS and LDFLAGS found in #cgo lines, and a map of file
 // hashes of the accessed C header files. If there is one or more error, it
 // returns these in the []error slice but still modifies the AST.
-func Process(files []*ast.File, dir, importPath string, fset *token.FileSet, cflags []string, clangHeaders string) (*ast.File, []string, []string, []string, map[string][]byte, []error) {
+func Process(files []*ast.File, dir, importPath string, fset *token.FileSet, cflags []string) (*ast.File, []string, []string, []string, map[string][]byte, []error) {
 	p := &cgoPackage{
 		currentDir:      dir,
 		importPath:      importPath,
@@ -292,9 +292,6 @@ func Process(files []*ast.File, dir, importPath string, fset *token.FileSet, cfl
 	// have better alternatives anyway.
 	cflagsForCGo := append([]string{"-D_FORTIFY_SOURCE=0"}, cflags...)
 	cflagsForCGo = append(cflagsForCGo, p.cflags...)
-	if clangHeaders != "" {
-		cflagsForCGo = append(cflagsForCGo, "-isystem", clangHeaders)
-	}
 
 	// Retrieve types such as C.int, C.longlong, etc from C.
 	p.newCGoFile(nil, -1).readNames(builtinAliasTypedefs, cflagsForCGo, "", func(names map[string]clangCursor) {

cgo/cgo_test.go

@@ -55,7 +55,7 @@ func TestCGo(t *testing.T) {
 			}
 
 			// Process the AST with CGo.
-			cgoAST, _, _, _, _, cgoErrors := Process([]*ast.File{f}, "testdata", "main", fset, cflags, "")
+			cgoAST, _, _, _, _, cgoErrors := Process([]*ast.File{f}, "testdata", "main", fset, cflags)
 
 			// Check the AST for type errors.
 			var typecheckErrors []error

compileopts/config.go

@@ -19,7 +19,6 @@ type Config struct {
 	Options        *Options
 	Target         *TargetSpec
 	GoMinorVersion int
-	ClangHeaders   string // Clang built-in header include path
 	TestConfig     TestConfig
 }
 
@@ -259,11 +258,19 @@ func (c *Config) DefaultBinaryExtension() string {
 
 // CFlags returns the flags to pass to the C compiler. This is necessary for CGo
 // preprocessing.
-func (c *Config) CFlags() []string {
+func (c *Config) CFlags(libclang bool) []string {
 	var cflags []string
 	for _, flag := range c.Target.CFlags {
 		cflags = append(cflags, strings.ReplaceAll(flag, "{root}", goenv.Get("TINYGOROOT")))
 	}
+	resourceDir := goenv.ClangResourceDir(libclang)
+	if resourceDir != "" {
+		// The resoure directory contains the built-in clang headers like
+		// stdbool.h, stdint.h, float.h, etc.
+		// It is left empty if we're using an external compiler (that already
+		// knows these headers).
+		cflags = append(cflags, "-resource-dir="+resourceDir)
+	}
 	switch c.Target.Libc {
 	case "darwin-libSystem":
 		root := goenv.Get("TINYGOROOT")
@@ -275,8 +282,8 @@ func (c *Config) CFlags() []string {
 		picolibcDir := filepath.Join(root, "lib", "picolibc", "newlib", "libc")
 		path, _ := c.LibcPath("picolibc")
 		cflags = append(cflags,
-			"--sysroot="+path,
+			"-nostdlibinc",
-			"-isystem", filepath.Join(path, "include"), // necessary for Xtensa
+			"-isystem", filepath.Join(path, "include"),
 			"-isystem", filepath.Join(picolibcDir, "include"),
 			"-isystem", filepath.Join(picolibcDir, "tinystdio"),
 		)

compiler/compiler_test.go

@@ -243,7 +243,7 @@ func testCompilePackage(t *testing.T, options *compileopts.Options, file string)
 	defer machine.Dispose()
 
 	// Load entire program AST into memory.
-	lprogram, err := loader.Load(config, "./testdata/"+file, config.ClangHeaders, types.Config{
+	lprogram, err := loader.Load(config, "./testdata/"+file, types.Config{
 		Sizes: Sizes(machine),
 	})
 	if err != nil {

goenv/goenv.go

@@ -14,6 +14,8 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+
+	"tinygo.org/x/go-llvm"
 )
 
 // Keys is a slice of all available environment variable keys.
@@ -33,6 +35,9 @@ func init() {
 	}
 }
 
+// Set to true if we're linking statically against LLVM.
+var hasBuiltinTools = false
+
 // TINYGOROOT is the path to the final location for checking tinygo files. If
 // unset (by a -X ldflag), then sourceDir() will fallback to the original build
 // directory.
@@ -284,3 +289,70 @@ func isSourceDir(root string) bool {
 	_, err = os.Stat(filepath.Join(root, "src/device/arm/arm.go"))
 	return err == nil
 }
+
+// ClangResourceDir returns the clang resource dir if available. This is the
+// -resource-dir flag. If it isn't available, an empty string is returned and
+// -resource-dir should be left unset.
+// The libclang flag must be set if the resource dir is read for use by
+// libclang.
+// In that case, the resource dir is always returned (even when linking
+// dynamically against LLVM) because libclang always needs this directory.
+func ClangResourceDir(libclang bool) string {
+	if !hasBuiltinTools && !libclang {
+		// Using external tools, so the resource dir doesn't need to be
+		// specified. Clang knows where to find it.
+		return ""
+	}
+
+	// Check whether we're running from a TinyGo release directory.
+	// This is the case for release binaries on GitHub.
+	root := Get("TINYGOROOT")
+	releaseHeaderDir := filepath.Join(root, "lib", "clang")
+	if _, err := os.Stat(releaseHeaderDir); !errors.Is(err, fs.ErrNotExist) {
+		return releaseHeaderDir
+	}
+
+	if hasBuiltinTools {
+		// We are statically linked to LLVM.
+		// Check whether we're running from the source directory.
+		// This typically happens when TinyGo was built using `make` as part of
+		// development.
+		llvmMajor := strings.Split(llvm.Version, ".")[0]
+		buildResourceDir := filepath.Join(root, "llvm-build", "lib", "clang", llvmMajor)
+		if _, err := os.Stat(buildResourceDir); !errors.Is(err, fs.ErrNotExist) {
+			return buildResourceDir
+		}
+	} else {
+		// We use external tools, either when installed using `go install` or
+		// when packaged in a Linux distribution (Linux distros typically prefer
+		// dynamic linking).
+		// Try to detect the system clang resources directory.
+		resourceDir := findSystemClangResources(root)
+		if resourceDir != "" {
+			return resourceDir
+		}
+	}
+
+	// Resource directory not found.
+	return ""
+}
+
+// Find the Clang resource dir on this particular system.
+// Return the empty string when they aren't found.
+func findSystemClangResources(TINYGOROOT string) string {
+	llvmMajor := strings.Split(llvm.Version, ".")[0]
+
+	switch runtime.GOOS {
+	case "linux", "android":
+		// Header files are typically stored in /usr/lib/clang/<version>/include.
+		// Tested on Fedora 39, Debian 12, and Arch Linux.
+		path := filepath.Join("/usr/lib/clang", llvmMajor)
+		_, err := os.Stat(filepath.Join(path, "include", "stdint.h"))
+		if err == nil {
+			return path
+		}
+	}
+
+	// Could not find it.
+	return ""
+}

goenv/tools-builtin.go

@@ -0,0 +1,7 @@
+//go:build byollvm
+
+package goenv
+
+func init() {
+	hasBuiltinTools = true
+}

loader/loader.go

@@ -29,11 +29,10 @@ import (
 
 // Program holds all packages and some metadata about the program as a whole.
 type Program struct {
-	config       *compileopts.Config
+	config      *compileopts.Config
-	clangHeaders string
+	typeChecker types.Config
-	typeChecker  types.Config
+	goroot      string // synthetic GOROOT
-	goroot       string // synthetic GOROOT
+	workingDir  string
-	workingDir   string
 
 	Packages map[string]*Package
 	sorted   []*Package
@@ -103,7 +102,7 @@ type EmbedFile struct {
 // Load loads the given package with all dependencies (including the runtime
 // package). Call .Parse() afterwards to parse all Go files (including CGo
 // processing, if necessary).
-func Load(config *compileopts.Config, inputPkg string, clangHeaders string, typeChecker types.Config) (*Program, error) {
+func Load(config *compileopts.Config, inputPkg string, typeChecker types.Config) (*Program, error) {
 	goroot, err := GetCachedGoroot(config)
 	if err != nil {
 		return nil, err
@@ -118,13 +117,12 @@ func Load(config *compileopts.Config, inputPkg string, clangHeaders string, type
 		}
 	}
 	p := &Program{
-		config:       config,
+		config:      config,
-		clangHeaders: clangHeaders,
+		typeChecker: typeChecker,
-		typeChecker:  typeChecker,
+		goroot:      goroot,
-		goroot:       goroot,
+		workingDir:  wd,
-		workingDir:   wd,
+		Packages:    make(map[string]*Package),
-		Packages:     make(map[string]*Package),
+		fset:        token.NewFileSet(),
-		fset:         token.NewFileSet(),
 	}
 
 	// List the dependencies of this package, in raw JSON format.
@@ -438,9 +436,9 @@ func (p *Package) parseFiles() ([]*ast.File, error) {
 	// to call cgo.Process in that case as it will only cause issues.
 	if len(p.CgoFiles) != 0 && len(files) != 0 {
 		var initialCFlags []string
-		initialCFlags = append(initialCFlags, p.program.config.CFlags()...)
+		initialCFlags = append(initialCFlags, p.program.config.CFlags(true)...)
 		initialCFlags = append(initialCFlags, "-I"+p.Dir)
-		generated, headerCode, cflags, ldflags, accessedFiles, errs := cgo.Process(files, p.program.workingDir, p.ImportPath, p.program.fset, initialCFlags, p.program.clangHeaders)
+		generated, headerCode, cflags, ldflags, accessedFiles, errs := cgo.Process(files, p.program.workingDir, p.ImportPath, p.program.fset, initialCFlags)
 		p.CFlags = append(initialCFlags, cflags...)
 		p.CGoHeaders = headerCode
 		for path, hash := range accessedFiles {

transform/transform_test.go

@@ -145,7 +145,7 @@ func compileGoFileForTesting(t *testing.T, filename string) llvm.Module {
 	defer machine.Dispose()
 
 	// Load entire program AST into memory.
-	lprogram, err := loader.Load(config, filename, config.ClangHeaders, types.Config{
+	lprogram, err := loader.Load(config, filename, types.Config{
 		Sizes: compiler.Sizes(machine),
 	})
 	if err != nil {