cgo: refactor; support multiple cgo files in a single package

This is a big commit that does a few things:

  * It moves CGo processing into a separate package. It never really
    belonged in the loader package, and certainly not now that the
    loader package may be refactored into a driver package.
  * It adds support for multiple CGo files (files that import package
    "C") in a single package. Previously, this led to multiple
    definition errors in the Go typecheck phase because certain C
    symbols were defined multiple times in all the files. Now it
    generates a new fake AST that defines these, to avoid multiple
    definition errors.
  * It improves debug info in a few edge cases that are probably not
    relevant outside of bugs in cgo itself.

Makefile

@@ -32,7 +32,7 @@ CGO_LDFLAGS=-L$(LLVM_BUILDDIR)/lib $(CLANG_LIBS) $(LLD_LIBS) $(shell $(LLVM_BUIL
 clean:
 	@rm -rf build
 
-FMT_PATHS = ./*.go compiler interp ir loader src/device/arm src/examples src/machine src/os src/reflect src/runtime src/sync src/syscall
+FMT_PATHS = ./*.go cgo compiler interp ir loader src/device/arm src/examples src/machine src/os src/reflect src/runtime src/sync src/syscall
 fmt:
 	@gofmt -l -w $(FMT_PATHS)
 fmt-check:

cgo/cgo.go

@@ -1,7 +1,15 @@
-package loader
+// Package cgo implements CGo by modifying a loaded AST. It does this by parsing
+// the `import "C"` statements found in the source code with libclang and
+// generating stub function and global declarations.
+//
+// There are a few advantages to modifying the AST directly instead of doing CGo
+// as a preprocessing step, with the main advantage being that debug information
+// is kept intact as much as possible.
+package cgo
 
 // This file extracts the `import "C"` statement from the source and modifies
-// the AST for Cgo. It does not use libclang directly (see libclang.go).
+// the AST for CCo. It does not use libclang directly: see libclang.go for the C
+// source file parsing.
 
 import (
 	"go/ast"
@@ -13,25 +21,35 @@ import (
 	"golang.org/x/tools/go/ast/astutil"
 )
 
-// fileInfo holds all Cgo-related information of a given *ast.File.
+// cgoPackage holds all CCo-related information of a package.
-type fileInfo struct {
+type cgoPackage struct {
-	*ast.File
+	generated       *ast.File
-	*Package
+	generatedPos    token.Pos
-	filename        string
+	errors          []error
-	constants       map[string]*ast.BasicLit
+	dir             string
-	functions       map[string]*functionInfo
+	fset            *token.FileSet
-	globals         map[string]*globalInfo
+	tokenFiles      map[string]*token.File
-	typedefs        map[string]*typedefInfo
-	elaboratedTypes map[string]ast.Expr
-	importCPos      token.Pos
 	missingSymbols  map[string]struct{}
+	constants       map[string]constantInfo
+	functions       map[string]*functionInfo
+	globals         map[string]globalInfo
+	typedefs        map[string]*typedefInfo
+	elaboratedTypes map[string]*elaboratedTypeInfo
 }
 
-// functionInfo stores some information about a Cgo function found by libclang
+// constantInfo stores some information about a CGo constant found by libclang
+// and declared in the Go AST.
+type constantInfo struct {
+	expr *ast.BasicLit
+	pos  token.Pos
+}
+
+// functionInfo stores some information about a CCo function found by libclang
 // and declared in the AST.
 type functionInfo struct {
 	args    []paramInfo
 	results *ast.FieldList
+	pos     token.Pos
 }
 
 // paramInfo is a parameter of a Cgo function (see functionInfo).
@@ -43,11 +61,20 @@ type paramInfo struct {
 // typedefInfo contains information about a single typedef in C.
 type typedefInfo struct {
 	typeExpr ast.Expr
+	pos      token.Pos
+}
+
+// elaboratedTypeInfo contains some information about an elaborated type
+// (struct, union) found in the C AST.
+type elaboratedTypeInfo struct {
+	typeExpr ast.Expr
+	pos      token.Pos
 }
 
 // globalInfo contains information about a declared global variable in C.
 type globalInfo struct {
 	typeExpr ast.Expr
+	pos      token.Pos
 }
 
 // cgoAliases list type aliases between Go and C, for types that are equivalent
@@ -64,9 +91,9 @@ var cgoAliases = map[string]string{
 	"C.uintptr_t": "uintptr",
 }
 
-// cgoBuiltinAliases are handled specially because they only exist on the Go
+// builtinAliases are handled specially because they only exist on the Go side
-// side of CGo, not on the CGo (they're prefixed with "_Cgo_" there).
+// of CGo, not on the CGo side (they're prefixed with "_Cgo_" there).
-var cgoBuiltinAliases = map[string]struct{}{
+var builtinAliases = map[string]struct{}{
 	"char":      struct{}{},
 	"schar":     struct{}{},
 	"uchar":     struct{}{},
@@ -97,111 +124,146 @@ typedef long long           _Cgo_longlong;
 typedef unsigned long long  _Cgo_ulonglong;
 `
 
-// processCgo extracts the `import "C"` statement from the AST, parses the
+// Process extracts `import "C"` statements from the AST, parses the comment
-// comment with libclang, and modifies the AST to use this information.
+// with libclang, and modifies the AST to use this information. It returns a
-func (p *Package) processCgo(filename string, f *ast.File, cflags []string) []error {
+// newly created *ast.File that should be added to the list of to-be-parsed
-	info := &fileInfo{
+// files. If there is one or more error, it returns these in the []error slice
-		File:            f,
+// but still modifies the AST.
-		Package:         p,
+func Process(files []*ast.File, dir string, fset *token.FileSet, cflags []string) (*ast.File, []error) {
-		filename:        filename,
+	p := &cgoPackage{
-		constants:       map[string]*ast.BasicLit{},
+		dir:             dir,
-		functions:       map[string]*functionInfo{},
+		fset:            fset,
-		globals:         map[string]*globalInfo{},
+		tokenFiles:      map[string]*token.File{},
-		typedefs:        map[string]*typedefInfo{},
-		elaboratedTypes: map[string]ast.Expr{},
 		missingSymbols:  map[string]struct{}{},
+		constants:       map[string]constantInfo{},
+		functions:       map[string]*functionInfo{},
+		globals:         map[string]globalInfo{},
+		typedefs:        map[string]*typedefInfo{},
+		elaboratedTypes: map[string]*elaboratedTypeInfo{},
+	}
+
+	// Add a new location for the following file.
+	generatedTokenPos := p.fset.AddFile(dir+"/!cgo.go", -1, 0)
+	generatedTokenPos.SetLines([]int{0})
+	p.generatedPos = generatedTokenPos.Pos(0)
+
+	// Construct a new in-memory AST for CGo declarations of this package.
+	unsafeImport := &ast.ImportSpec{
+		Path: &ast.BasicLit{
+			ValuePos: p.generatedPos,
+			Kind:     token.STRING,
+			Value:    "\"unsafe\"",
+		},
+		EndPos: p.generatedPos,
+	}
+	p.generated = &ast.File{
+		Package: p.generatedPos,
+		Name: &ast.Ident{
+			NamePos: p.generatedPos,
+			Name:    files[0].Name.Name,
+		},
+		Decls: []ast.Decl{
+			&ast.GenDecl{
+				TokPos: p.generatedPos,
+				Tok:    token.IMPORT,
+				Specs: []ast.Spec{
+					unsafeImport,
+				},
+			},
+		},
+		Imports: []*ast.ImportSpec{unsafeImport},
 	}
 
 	// Find all C.* symbols.
-	f = astutil.Apply(f, info.findMissingCGoNames, nil).(*ast.File)
+	for _, f := range files {
-	for name := range cgoBuiltinAliases {
+		astutil.Apply(f, p.findMissingCGoNames, nil)
-		info.missingSymbols["_Cgo_"+name] = struct{}{}
+	}
+	for name := range builtinAliases {
+		p.missingSymbols["_Cgo_"+name] = struct{}{}
 	}
 
 	// Find `import "C"` statements in the file.
-	for i := 0; i < len(f.Decls); i++ {
+	for _, f := range files {
-		decl := f.Decls[i]
+		for i := 0; i < len(f.Decls); i++ {
-		genDecl, ok := decl.(*ast.GenDecl)
+			decl := f.Decls[i]
-		if !ok {
+			genDecl, ok := decl.(*ast.GenDecl)
-			continue
+			if !ok {
-		}
+				continue
-		if len(genDecl.Specs) != 1 {
+			}
-			continue
+			if len(genDecl.Specs) != 1 {
-		}
+				continue
-		spec, ok := genDecl.Specs[0].(*ast.ImportSpec)
+			}
-		if !ok {
+			spec, ok := genDecl.Specs[0].(*ast.ImportSpec)
-			continue
+			if !ok {
-		}
+				continue
-		path, err := strconv.Unquote(spec.Path.Value)
+			}
-		if err != nil {
+			path, err := strconv.Unquote(spec.Path.Value)
-			panic("could not parse import path: " + err.Error())
+			if err != nil {
-		}
+				panic("could not parse import path: " + err.Error())
-		if path != "C" {
+			}
-			continue
+			if path != "C" {
-		}
+				continue
-		cgoComment := genDecl.Doc.Text()
+			}
+			cgoComment := genDecl.Doc.Text()
 
-		// Stored for later use by generated functions, to use a somewhat sane
+			pos := genDecl.Pos()
-		// source location.
+			if genDecl.Doc != nil {
-		info.importCPos = spec.Path.ValuePos
+				pos = genDecl.Doc.Pos()
+			}
+			position := fset.PositionFor(pos, true)
+			p.parseFragment(cgoComment+cgoTypes, cflags, position.Filename, position.Line)
 
-		pos := genDecl.Pos()
+			// Remove this import declaration.
-		if genDecl.Doc != nil {
+			f.Decls = append(f.Decls[:i], f.Decls[i+1:]...)
-			pos = genDecl.Doc.Pos()
+			i--
-		}
-		position := info.fset.PositionFor(pos, true)
-		errs := info.parseFragment(cgoComment+cgoTypes, cflags, position.Filename, position.Line)
-		if errs != nil {
-			return errs
 		}
 
-		// Remove this import declaration.
+		// Print the AST, for debugging.
-		f.Decls = append(f.Decls[:i], f.Decls[i+1:]...)
+		//ast.Print(fset, f)
-		i--
 	}
 
-	// Print the AST, for debugging.
-	//ast.Print(p.fset, f)
-
 	// Declare functions found by libclang.
-	info.addFuncDecls()
+	p.addFuncDecls()
 
 	// Declare stub function pointer values found by libclang.
-	info.addFuncPtrDecls()
+	p.addFuncPtrDecls()
 
 	// Declare globals found by libclang.
-	info.addConstDecls()
+	p.addConstDecls()
 
 	// Declare globals found by libclang.
-	info.addVarDecls()
+	p.addVarDecls()
 
 	// Forward C types to Go types (like C.uint32_t -> uint32).
-	info.addTypeAliases()
+	p.addTypeAliases()
 
 	// Add type declarations for C types, declared using typedef in C.
-	info.addTypedefs()
+	p.addTypedefs()
 
 	// Add elaborated types for C structs and unions.
-	info.addElaboratedTypes()
+	p.addElaboratedTypes()
 
 	// Patch the AST to use the declared types and functions.
-	f = astutil.Apply(f, info.walker, nil).(*ast.File)
+	for _, f := range files {
+		astutil.Apply(f, p.walker, nil)
+	}
 
-	return nil
+	// Print the newly generated in-memory AST, for debugging.
+	//ast.Print(fset, p.generated)
+
+	return p.generated, p.errors
 }
 
 // addFuncDecls adds the C function declarations found by libclang in the
 // comment above the `import "C"` statement.
-func (info *fileInfo) addFuncDecls() {
+func (p *cgoPackage) addFuncDecls() {
-	// TODO: replace all uses of importCPos with the real locations from
+	names := make([]string, 0, len(p.functions))
-	// libclang.
+	for name := range p.functions {
-	names := make([]string, 0, len(info.functions))
-	for name := range info.functions {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		fn := info.functions[name]
+		fn := p.functions[name]
 		obj := &ast.Object{
 			Kind: ast.Fun,
 			Name: "C." + name,
@@ -209,16 +271,16 @@ func (info *fileInfo) addFuncDecls() {
 		args := make([]*ast.Field, len(fn.args))
 		decl := &ast.FuncDecl{
 			Name: &ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: fn.pos,
 				Name:    "C." + name,
 				Obj:     obj,
 			},
 			Type: &ast.FuncType{
-				Func: info.importCPos,
+				Func: fn.pos,
 				Params: &ast.FieldList{
-					Opening: info.importCPos,
+					Opening: fn.pos,
 					List:    args,
-					Closing: info.importCPos,
+					Closing: fn.pos,
 				},
 				Results: fn.results,
 			},
@@ -228,7 +290,7 @@ func (info *fileInfo) addFuncDecls() {
 			args[i] = &ast.Field{
 				Names: []*ast.Ident{
 					&ast.Ident{
-						NamePos: info.importCPos,
+						NamePos: fn.pos,
 						Name:    arg.name,
 						Obj: &ast.Object{
 							Kind: ast.Var,
@@ -240,7 +302,7 @@ func (info *fileInfo) addFuncDecls() {
 				Type: arg.typeExpr,
 			}
 		}
-		info.Decls = append(info.Decls, decl)
+		p.generated.Decls = append(p.generated.Decls, decl)
 	}
 }
 
@@ -253,39 +315,40 @@ func (info *fileInfo) addFuncDecls() {
 //         C.mul unsafe.Pointer
 //         // ...
 //     )
-func (info *fileInfo) addFuncPtrDecls() {
+func (p *cgoPackage) addFuncPtrDecls() {
-	if len(info.functions) == 0 {
+	if len(p.functions) == 0 {
 		return
 	}
 	gen := &ast.GenDecl{
-		TokPos: info.importCPos,
+		TokPos: token.NoPos,
 		Tok:    token.VAR,
-		Lparen: info.importCPos,
+		Lparen: token.NoPos,
-		Rparen: info.importCPos,
+		Rparen: token.NoPos,
 	}
-	names := make([]string, 0, len(info.functions))
+	names := make([]string, 0, len(p.functions))
-	for name := range info.functions {
+	for name := range p.functions {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
+		fn := p.functions[name]
 		obj := &ast.Object{
 			Kind: ast.Typ,
 			Name: "C." + name + "$funcaddr",
 		}
 		valueSpec := &ast.ValueSpec{
 			Names: []*ast.Ident{&ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: fn.pos,
 				Name:    "C." + name + "$funcaddr",
 				Obj:     obj,
 			}},
 			Type: &ast.SelectorExpr{
 				X: &ast.Ident{
-					NamePos: info.importCPos,
+					NamePos: fn.pos,
 					Name:    "unsafe",
 				},
 				Sel: &ast.Ident{
-					NamePos: info.importCPos,
+					NamePos: fn.pos,
 					Name:    "Pointer",
 				},
 			},
@@ -293,7 +356,7 @@ func (info *fileInfo) addFuncPtrDecls() {
 		obj.Decl = valueSpec
 		gen.Specs = append(gen.Specs, valueSpec)
 	}
-	info.Decls = append(info.Decls, gen)
+	p.generated.Decls = append(p.generated.Decls, gen)
 }
 
 // addConstDecls declares external C constants in the Go source.
@@ -304,39 +367,39 @@ func (info *fileInfo) addFuncPtrDecls() {
 //         C.CONST_FLOAT = 5.8
 //         // ...
 //     )
-func (info *fileInfo) addConstDecls() {
+func (p *cgoPackage) addConstDecls() {
-	if len(info.constants) == 0 {
+	if len(p.constants) == 0 {
 		return
 	}
 	gen := &ast.GenDecl{
-		TokPos: info.importCPos,
+		TokPos: token.NoPos,
 		Tok:    token.CONST,
-		Lparen: info.importCPos,
+		Lparen: token.NoPos,
-		Rparen: info.importCPos,
+		Rparen: token.NoPos,
 	}
-	names := make([]string, 0, len(info.constants))
+	names := make([]string, 0, len(p.constants))
-	for name := range info.constants {
+	for name := range p.constants {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		constVal := info.constants[name]
+		constVal := p.constants[name]
 		obj := &ast.Object{
 			Kind: ast.Con,
 			Name: "C." + name,
 		}
 		valueSpec := &ast.ValueSpec{
 			Names: []*ast.Ident{&ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: constVal.pos,
 				Name:    "C." + name,
 				Obj:     obj,
 			}},
-			Values: []ast.Expr{constVal},
+			Values: []ast.Expr{constVal.expr},
 		}
 		obj.Decl = valueSpec
 		gen.Specs = append(gen.Specs, valueSpec)
 	}
-	info.Decls = append(info.Decls, gen)
+	p.generated.Decls = append(p.generated.Decls, gen)
 }
 
 // addVarDecls declares external C globals in the Go source.
@@ -347,30 +410,30 @@ func (info *fileInfo) addConstDecls() {
 //         C.globalBool bool
 //         // ...
 //     )
-func (info *fileInfo) addVarDecls() {
+func (p *cgoPackage) addVarDecls() {
-	if len(info.globals) == 0 {
+	if len(p.globals) == 0 {
 		return
 	}
 	gen := &ast.GenDecl{
-		TokPos: info.importCPos,
+		TokPos: token.NoPos,
 		Tok:    token.VAR,
-		Lparen: info.importCPos,
+		Lparen: token.NoPos,
-		Rparen: info.importCPos,
+		Rparen: token.NoPos,
 	}
-	names := make([]string, 0, len(info.globals))
+	names := make([]string, 0, len(p.globals))
-	for name := range info.globals {
+	for name := range p.globals {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		global := info.globals[name]
+		global := p.globals[name]
 		obj := &ast.Object{
 			Kind: ast.Var,
 			Name: "C." + name,
 		}
 		valueSpec := &ast.ValueSpec{
 			Names: []*ast.Ident{&ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: global.pos,
 				Name:    "C." + name,
 				Obj:     obj,
 			}},
@@ -379,7 +442,7 @@ func (info *fileInfo) addVarDecls() {
 		obj.Decl = valueSpec
 		gen.Specs = append(gen.Specs, valueSpec)
 	}
-	info.Decls = append(info.Decls, gen)
+	p.generated.Decls = append(p.generated.Decls, gen)
 }
 
 // addTypeAliases aliases some built-in Go types with their equivalent C types.
@@ -390,17 +453,17 @@ func (info *fileInfo) addVarDecls() {
 //         C.int16_t = int16
 //         // ...
 //     )
-func (info *fileInfo) addTypeAliases() {
+func (p *cgoPackage) addTypeAliases() {
 	aliasKeys := make([]string, 0, len(cgoAliases))
 	for key := range cgoAliases {
 		aliasKeys = append(aliasKeys, key)
 	}
 	sort.Strings(aliasKeys)
 	gen := &ast.GenDecl{
-		TokPos: info.importCPos,
+		TokPos: token.NoPos,
 		Tok:    token.TYPE,
-		Lparen: info.importCPos,
+		Lparen: token.NoPos,
-		Rparen: info.importCPos,
+		Rparen: token.NoPos,
 	}
 	for _, typeName := range aliasKeys {
 		goTypeName := cgoAliases[typeName]
@@ -410,37 +473,37 @@ func (info *fileInfo) addTypeAliases() {
 		}
 		typeSpec := &ast.TypeSpec{
 			Name: &ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: token.NoPos,
 				Name:    typeName,
 				Obj:     obj,
 			},
-			Assign: info.importCPos,
+			Assign: p.generatedPos,
 			Type: &ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: token.NoPos,
 				Name:    goTypeName,
 			},
 		}
 		obj.Decl = typeSpec
 		gen.Specs = append(gen.Specs, typeSpec)
 	}
-	info.Decls = append(info.Decls, gen)
+	p.generated.Decls = append(p.generated.Decls, gen)
 }
 
-func (info *fileInfo) addTypedefs() {
+func (p *cgoPackage) addTypedefs() {
-	if len(info.typedefs) == 0 {
+	if len(p.typedefs) == 0 {
 		return
 	}
 	gen := &ast.GenDecl{
-		TokPos: info.importCPos,
+		TokPos: token.NoPos,
 		Tok:    token.TYPE,
 	}
-	names := make([]string, 0, len(info.typedefs))
+	names := make([]string, 0, len(p.typedefs))
-	for name := range info.typedefs {
+	for name := range p.typedefs {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		typedef := info.typedefs[name]
+		typedef := p.typedefs[name]
 		typeName := "C." + name
 		isAlias := true
 		if strings.HasPrefix(name, "_Cgo_") {
@@ -457,19 +520,19 @@ func (info *fileInfo) addTypedefs() {
 		}
 		typeSpec := &ast.TypeSpec{
 			Name: &ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: typedef.pos,
 				Name:    typeName,
 				Obj:     obj,
 			},
 			Type: typedef.typeExpr,
 		}
 		if isAlias {
-			typeSpec.Assign = info.importCPos
+			typeSpec.Assign = typedef.pos
 		}
 		obj.Decl = typeSpec
 		gen.Specs = append(gen.Specs, typeSpec)
 	}
-	info.Decls = append(info.Decls, gen)
+	p.generated.Decls = append(p.generated.Decls, gen)
 }
 
 // addElaboratedTypes adds C elaborated types as aliases. These are the "struct
@@ -477,21 +540,21 @@ func (info *fileInfo) addTypedefs() {
 //
 // See also:
 // https://en.cppreference.com/w/cpp/language/elaborated_type_specifier
-func (info *fileInfo) addElaboratedTypes() {
+func (p *cgoPackage) addElaboratedTypes() {
-	if len(info.elaboratedTypes) == 0 {
+	if len(p.elaboratedTypes) == 0 {
 		return
 	}
 	gen := &ast.GenDecl{
-		TokPos: info.importCPos,
+		TokPos: token.NoPos,
 		Tok:    token.TYPE,
 	}
-	names := make([]string, 0, len(info.elaboratedTypes))
+	names := make([]string, 0, len(p.elaboratedTypes))
-	for name := range info.elaboratedTypes {
+	for name := range p.elaboratedTypes {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		typ := info.elaboratedTypes[name]
+		typ := p.elaboratedTypes[name]
 		typeName := "C." + name
 		obj := &ast.Object{
 			Kind: ast.Typ,
@@ -499,22 +562,22 @@ func (info *fileInfo) addElaboratedTypes() {
 		}
 		typeSpec := &ast.TypeSpec{
 			Name: &ast.Ident{
-				NamePos: info.importCPos,
+				NamePos: typ.pos,
 				Name:    typeName,
 				Obj:     obj,
 			},
-			Type: typ,
+			Type: typ.typeExpr,
 		}
 		obj.Decl = typeSpec
 		gen.Specs = append(gen.Specs, typeSpec)
 	}
-	info.Decls = append(info.Decls, gen)
+	p.generated.Decls = append(p.generated.Decls, gen)
 }
 
 // findMissingCGoNames traverses the AST and finds all C.something names. Only
 // these symbols are extracted from the parsed C AST and converted to the Go
 // equivalent.
-func (info *fileInfo) findMissingCGoNames(cursor *astutil.Cursor) bool {
+func (p *cgoPackage) findMissingCGoNames(cursor *astutil.Cursor) bool {
 	switch node := cursor.Node().(type) {
 	case *ast.SelectorExpr:
 		x, ok := node.X.(*ast.Ident)
@@ -523,10 +586,10 @@ func (info *fileInfo) findMissingCGoNames(cursor *astutil.Cursor) bool {
 		}
 		if x.Name == "C" {
 			name := node.Sel.Name
-			if _, ok := cgoBuiltinAliases[name]; ok {
+			if _, ok := builtinAliases[name]; ok {
 				name = "_Cgo_" + name
 			}
-			info.missingSymbols[name] = struct{}{}
+			p.missingSymbols[name] = struct{}{}
 		}
 	}
 	return true
@@ -536,7 +599,7 @@ func (info *fileInfo) findMissingCGoNames(cursor *astutil.Cursor) bool {
 // expressions. Such expressions are impossible to write in Go (a dot cannot be
 // used in the middle of a name) so in practice all C identifiers live in a
 // separate namespace (no _Cgo_ hacks like in gc).
-func (info *fileInfo) walker(cursor *astutil.Cursor) bool {
+func (p *cgoPackage) walker(cursor *astutil.Cursor) bool {
 	switch node := cursor.Node().(type) {
 	case *ast.CallExpr:
 		fun, ok := node.Fun.(*ast.SelectorExpr)
@@ -547,7 +610,7 @@ func (info *fileInfo) walker(cursor *astutil.Cursor) bool {
 		if !ok {
 			return true
 		}
-		if _, ok := info.functions[fun.Sel.Name]; ok && x.Name == "C" {
+		if _, ok := p.functions[fun.Sel.Name]; ok && x.Name == "C" {
 			node.Fun = &ast.Ident{
 				NamePos: x.NamePos,
 				Name:    "C." + fun.Sel.Name,
@@ -560,7 +623,7 @@ func (info *fileInfo) walker(cursor *astutil.Cursor) bool {
 		}
 		if x.Name == "C" {
 			name := "C." + node.Sel.Name
-			if _, ok := info.functions[node.Sel.Name]; ok {
+			if _, ok := p.functions[node.Sel.Name]; ok {
 				name += "$funcaddr"
 			}
 			cursor.Replace(&ast.Ident{

cgo/libclang.go

@@ -1,4 +1,4 @@
-package loader
+package cgo
 
 // This file parses a fragment of C with libclang and stores the result for AST
 // modification. It does not touch the AST itself.
@@ -55,8 +55,8 @@ int tinygo_clang_struct_visitor(GoCXCursor c, GoCXCursor parent, CXClientData cl
 */
 import "C"
 
-// refMap stores references to types, used for clang_visitChildren.
+// storedRefs stores references to types, used for clang_visitChildren.
-var refMap RefMap
+var storedRefs refMap
 
 var diagnosticSeverity = [...]string{
 	C.CXDiagnostic_Ignored: "ignored",
@@ -66,7 +66,7 @@ var diagnosticSeverity = [...]string{
 	C.CXDiagnostic_Fatal:   "fatal",
 }
 
-func (info *fileInfo) parseFragment(fragment string, cflags []string, posFilename string, posLine int) []error {
+func (p *cgoPackage) parseFragment(fragment string, cflags []string, posFilename string, posLine int) {
 	index := C.clang_createIndex(0, 0)
 	defer C.clang_disposeIndex(index)
 
@@ -110,7 +110,6 @@ func (info *fileInfo) parseFragment(fragment string, cflags []string, posFilenam
 	defer C.clang_disposeTranslationUnit(unit)
 
 	if numDiagnostics := int(C.clang_getNumDiagnostics(unit)); numDiagnostics != 0 {
-		errs := []error{}
 		addDiagnostic := func(diagnostic C.CXDiagnostic) {
 			spelling := getString(C.clang_getDiagnosticSpelling(diagnostic))
 			severity := diagnosticSeverity[C.clang_getDiagnosticSeverity(diagnostic)]
@@ -122,12 +121,12 @@ func (info *fileInfo) parseFragment(fragment string, cflags []string, posFilenam
 			filename := getString(libclangFilename)
 			if filepath.IsAbs(filename) {
 				// Relative paths for readability, like other Go parser errors.
-				relpath, err := filepath.Rel(info.Program.Dir, filename)
+				relpath, err := filepath.Rel(p.dir, filename)
 				if err == nil {
 					filename = relpath
 				}
 			}
-			errs = append(errs, &scanner.Error{
+			p.errors = append(p.errors, &scanner.Error{
 				Pos: token.Position{
 					Filename: filename,
 					Offset:   0, // not provided by clang_getPresumedLocation
@@ -147,26 +146,24 @@ func (info *fileInfo) parseFragment(fragment string, cflags []string, posFilenam
 				addDiagnostic(C.clang_getDiagnosticInSet(diagnostics, C.uint(j)))
 			}
 		}
-		return errs
+		return
 	}
 
-	ref := refMap.Put(info)
+	ref := storedRefs.Put(p)
-	defer refMap.Remove(ref)
+	defer storedRefs.Remove(ref)
 	cursor := C.tinygo_clang_getTranslationUnitCursor(unit)
 	C.tinygo_clang_visitChildren(cursor, C.CXCursorVisitor(C.tinygo_clang_globals_visitor), C.CXClientData(ref))
-
-	return nil
 }
 
 //export tinygo_clang_globals_visitor
 func tinygo_clang_globals_visitor(c, parent C.GoCXCursor, client_data C.CXClientData) C.int {
-	info := refMap.Get(unsafe.Pointer(client_data)).(*fileInfo)
+	p := storedRefs.Get(unsafe.Pointer(client_data)).(*cgoPackage)
 	kind := C.tinygo_clang_getCursorKind(c)
-	pos := info.getCursorPosition(c)
+	pos := p.getCursorPosition(c)
 	switch kind {
 	case C.CXCursor_FunctionDecl:
 		name := getString(C.tinygo_clang_getCursorSpelling(c))
-		if _, required := info.missingSymbols[name]; !required {
+		if _, required := p.missingSymbols[name]; !required {
 			return C.CXChildVisit_Continue
 		}
 		cursorType := C.tinygo_clang_getCursorType(c)
@@ -174,8 +171,10 @@ func tinygo_clang_globals_visitor(c, parent C.GoCXCursor, client_data C.CXClient
 			return C.CXChildVisit_Continue // not supported
 		}
 		numArgs := int(C.tinygo_clang_Cursor_getNumArguments(c))
-		fn := &functionInfo{}
+		fn := &functionInfo{
-		info.functions[name] = fn
+			pos: pos,
+		}
+		p.functions[name] = fn
 		for i := 0; i < numArgs; i++ {
 			arg := C.tinygo_clang_Cursor_getArgument(c, C.uint(i))
 			argName := getString(C.tinygo_clang_getCursorSpelling(arg))
@@ -185,7 +184,7 @@ func tinygo_clang_globals_visitor(c, parent C.GoCXCursor, client_data C.CXClient
 			}
 			fn.args = append(fn.args, paramInfo{
 				name:     argName,
-				typeExpr: info.makeASTType(argType, pos),
+				typeExpr: p.makeASTType(argType, pos),
 			})
 		}
 		resultType := C.tinygo_clang_getCursorResultType(c)
@@ -193,7 +192,7 @@ func tinygo_clang_globals_visitor(c, parent C.GoCXCursor, client_data C.CXClient
 			fn.results = &ast.FieldList{
 				List: []*ast.Field{
 					&ast.Field{
-						Type: info.makeASTType(resultType, pos),
+						Type: p.makeASTType(resultType, pos),
 					},
 				},
 			}
@@ -201,29 +200,30 @@ func tinygo_clang_globals_visitor(c, parent C.GoCXCursor, client_data C.CXClient
 	case C.CXCursor_StructDecl:
 		typ := C.tinygo_clang_getCursorType(c)
 		name := getString(C.tinygo_clang_getCursorSpelling(c))
-		if _, required := info.missingSymbols["struct_"+name]; !required {
+		if _, required := p.missingSymbols["struct_"+name]; !required {
 			return C.CXChildVisit_Continue
 		}
-		info.makeASTType(typ, pos)
+		p.makeASTType(typ, pos)
 	case C.CXCursor_TypedefDecl:
 		typedefType := C.tinygo_clang_getCursorType(c)
 		name := getString(C.clang_getTypedefName(typedefType))
-		if _, required := info.missingSymbols[name]; !required {
+		if _, required := p.missingSymbols[name]; !required {
 			return C.CXChildVisit_Continue
 		}
-		info.makeASTType(typedefType, pos)
+		p.makeASTType(typedefType, pos)
 	case C.CXCursor_VarDecl:
 		name := getString(C.tinygo_clang_getCursorSpelling(c))
-		if _, required := info.missingSymbols[name]; !required {
+		if _, required := p.missingSymbols[name]; !required {
 			return C.CXChildVisit_Continue
 		}
 		cursorType := C.tinygo_clang_getCursorType(c)
-		info.globals[name] = &globalInfo{
+		p.globals[name] = globalInfo{
-			typeExpr: info.makeASTType(cursorType, pos),
+			typeExpr: p.makeASTType(cursorType, pos),
+			pos:      pos,
 		}
 	case C.CXCursor_MacroDefinition:
 		name := getString(C.tinygo_clang_getCursorSpelling(c))
-		if _, required := info.missingSymbols[name]; !required {
+		if _, required := p.missingSymbols[name]; !required {
 			return C.CXChildVisit_Continue
 		}
 		sourceRange := C.tinygo_clang_getCursorExtent(c)
@@ -266,12 +266,12 @@ func tinygo_clang_globals_visitor(c, parent C.GoCXCursor, client_data C.CXClient
 		// https://en.cppreference.com/w/cpp/language/integer_literal
 		if value[0] == '"' {
 			// string constant
-			info.constants[name] = &ast.BasicLit{pos, token.STRING, value}
+			p.constants[name] = constantInfo{&ast.BasicLit{pos, token.STRING, value}, pos}
 			return C.CXChildVisit_Continue
 		}
 		if value[0] == '\'' {
 			// char constant
-			info.constants[name] = &ast.BasicLit{pos, token.CHAR, value}
+			p.constants[name] = constantInfo{&ast.BasicLit{pos, token.CHAR, value}, pos}
 			return C.CXChildVisit_Continue
 		}
 		// assume it's a number (int or float)
@@ -289,15 +289,15 @@ func tinygo_clang_globals_visitor(c, parent C.GoCXCursor, client_data C.CXClient
 		switch nonnum {
 		case 0:
 			// no non-number found, must be an integer
-			info.constants[name] = &ast.BasicLit{pos, token.INT, value}
+			p.constants[name] = constantInfo{&ast.BasicLit{pos, token.INT, value}, pos}
 		case 'x', 'X':
 			// hex integer constant
 			// TODO: may also be a floating point number per C++17.
-			info.constants[name] = &ast.BasicLit{pos, token.INT, value}
+			p.constants[name] = constantInfo{&ast.BasicLit{pos, token.INT, value}, pos}
 		case '.', 'e':
 			// float constant
 			value = strings.TrimRight(value, "fFlL")
-			info.constants[name] = &ast.BasicLit{pos, token.FLOAT, value}
+			p.constants[name] = constantInfo{&ast.BasicLit{pos, token.FLOAT, value}, pos}
 		default:
 			// unknown type, ignore
 		}
@@ -315,7 +315,7 @@ func getString(clangString C.CXString) (s string) {
 // getCursorPosition returns a usable token.Pos from a libclang cursor. If the
 // file for this cursor has not been seen before, it is read from libclang
 // (which already has the file in memory) and added to the token.FileSet.
-func (info *fileInfo) getCursorPosition(cursor C.GoCXCursor) token.Pos {
+func (p *cgoPackage) getCursorPosition(cursor C.GoCXCursor) token.Pos {
 	location := C.tinygo_clang_getCursorLocation(cursor)
 	var file C.CXFile
 	var line C.unsigned
@@ -327,7 +327,7 @@ func (info *fileInfo) getCursorPosition(cursor C.GoCXCursor) token.Pos {
 		return token.NoPos
 	}
 	filename := getString(C.clang_getFileName(file))
-	if _, ok := info.tokenFiles[filename]; !ok {
+	if _, ok := p.tokenFiles[filename]; !ok {
 		// File has not been seen before in this package, add line information
 		// now by reading the file from libclang.
 		tu := C.tinygo_clang_Cursor_getTranslationUnit(cursor)
@@ -340,16 +340,16 @@ func (info *fileInfo) getCursorPosition(cursor C.GoCXCursor) token.Pos {
 				lines = append(lines, i+1)
 			}
 		}
-		f := info.fset.AddFile(filename, -1, int(size))
+		f := p.fset.AddFile(filename, -1, int(size))
 		f.SetLines(lines)
-		info.tokenFiles[filename] = f
+		p.tokenFiles[filename] = f
 	}
-	return info.tokenFiles[filename].Pos(int(offset))
+	return p.tokenFiles[filename].Pos(int(offset))
 }
 
 // makeASTType return the ast.Expr for the given libclang type. In other words,
 // it converts a libclang type to a type in the Go AST.
-func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
+func (p *cgoPackage) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 	var typeName string
 	switch typ.kind {
 	case C.CXType_Char_S, C.CXType_Char_U:
@@ -410,7 +410,7 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 		}
 		return &ast.StarExpr{
 			Star: pos,
-			X:    info.makeASTType(pointeeType, pos),
+			X:    p.makeASTType(pointeeType, pos),
 		}
 	case C.CXType_ConstantArray:
 		return &ast.ArrayType{
@@ -420,7 +420,7 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 				Kind:     token.INT,
 				Value:    strconv.FormatInt(int64(C.clang_getArraySize(typ)), 10),
 			},
-			Elt: info.makeASTType(C.clang_getElementType(typ), pos),
+			Elt: p.makeASTType(C.clang_getElementType(typ), pos),
 		}
 	case C.CXType_FunctionProto:
 		// Be compatible with gc, which uses the *[0]byte type for function
@@ -441,11 +441,11 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 		}
 	case C.CXType_Typedef:
 		name := getString(C.clang_getTypedefName(typ))
-		if _, ok := info.typedefs[name]; !ok {
+		if _, ok := p.typedefs[name]; !ok {
-			info.typedefs[name] = nil // don't recurse
+			p.typedefs[name] = nil // don't recurse
 			c := C.tinygo_clang_getTypeDeclaration(typ)
 			underlyingType := C.tinygo_clang_getTypedefDeclUnderlyingType(c)
-			expr := info.makeASTType(underlyingType, pos)
+			expr := p.makeASTType(underlyingType, pos)
 			if strings.HasPrefix(name, "_Cgo_") {
 				expr := expr.(*ast.Ident)
 				typeSize := C.clang_Type_getSizeOf(underlyingType)
@@ -487,8 +487,9 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 					}
 				}
 			}
-			info.typedefs[name] = &typedefInfo{
+			p.typedefs[name] = &typedefInfo{
 				typeExpr: expr,
+				pos:      pos,
 			}
 		}
 		return &ast.Ident{
@@ -499,7 +500,7 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 		underlying := C.clang_Type_getNamedType(typ)
 		switch underlying.kind {
 		case C.CXType_Record:
-			return info.makeASTType(underlying, pos)
+			return p.makeASTType(underlying, pos)
 		default:
 			panic("unknown elaborated type")
 		}
@@ -515,23 +516,26 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 		default:
 			panic("unknown record declaration")
 		}
-		if _, ok := info.elaboratedTypes[cgoName]; !ok {
+		if _, ok := p.elaboratedTypes[cgoName]; !ok {
-			info.elaboratedTypes[cgoName] = nil // predeclare (to avoid endless recursion)
+			p.elaboratedTypes[cgoName] = nil // predeclare (to avoid endless recursion)
 			fieldList := &ast.FieldList{
 				Opening: pos,
 				Closing: pos,
 			}
-			ref := refMap.Put(struct {
+			ref := storedRefs.Put(struct {
 				fieldList *ast.FieldList
-				info      *fileInfo
+				pkg       *cgoPackage
-			}{fieldList, info})
+			}{fieldList, p})
-			defer refMap.Remove(ref)
+			defer storedRefs.Remove(ref)
 			C.tinygo_clang_visitChildren(cursor, C.CXCursorVisitor(C.tinygo_clang_struct_visitor), C.CXClientData(ref))
 			switch C.tinygo_clang_getCursorKind(cursor) {
 			case C.CXCursor_StructDecl:
-				info.elaboratedTypes[cgoName] = &ast.StructType{
+				p.elaboratedTypes[cgoName] = &elaboratedTypeInfo{
-					Struct: pos,
+					typeExpr: &ast.StructType{
-					Fields: fieldList,
+						Struct: pos,
+						Fields: fieldList,
+					},
+					pos: pos,
 				}
 			case C.CXCursor_UnionDecl:
 				if len(fieldList.List) > 1 {
@@ -561,9 +565,12 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 					}
 					fieldList.List = append([]*ast.Field{unionMarker}, fieldList.List...)
 				}
-				info.elaboratedTypes[cgoName] = &ast.StructType{
+				p.elaboratedTypes[cgoName] = &elaboratedTypeInfo{
-					Struct: pos,
+					typeExpr: &ast.StructType{
-					Fields: fieldList,
+						Struct: pos,
+						Fields: fieldList,
+					},
+					pos: pos,
 				}
 			default:
 				panic("unreachable")
@@ -587,23 +594,23 @@ func (info *fileInfo) makeASTType(typ C.CXType, pos token.Pos) ast.Expr {
 
 //export tinygo_clang_struct_visitor
 func tinygo_clang_struct_visitor(c, parent C.GoCXCursor, client_data C.CXClientData) C.int {
-	passed := refMap.Get(unsafe.Pointer(client_data)).(struct {
+	passed := storedRefs.Get(unsafe.Pointer(client_data)).(struct {
 		fieldList *ast.FieldList
-		info      *fileInfo
+		pkg       *cgoPackage
 	})
 	fieldList := passed.fieldList
-	info := passed.info
+	p := passed.pkg
 	if C.tinygo_clang_getCursorKind(c) != C.CXCursor_FieldDecl {
 		panic("expected field inside cursor")
 	}
 	name := getString(C.tinygo_clang_getCursorSpelling(c))
 	typ := C.tinygo_clang_getCursorType(c)
 	field := &ast.Field{
-		Type: info.makeASTType(typ, info.getCursorPosition(c)),
+		Type: p.makeASTType(typ, p.getCursorPosition(c)),
 	}
 	field.Names = []*ast.Ident{
 		&ast.Ident{
-			NamePos: info.getCursorPosition(c),
+			NamePos: p.getCursorPosition(c),
 			Name:    name,
 			Obj: &ast.Object{
 				Kind: ast.Var,

cgo/libclang_config.go

@@ -1,6 +1,6 @@
 // +build !byollvm
 
-package loader
+package cgo
 
 /*
 #cgo linux  CFLAGS: -I/usr/lib/llvm-8/include

cgo/sync.go

@@ -1,4 +1,4 @@
-package loader
+package cgo
 
 import (
 	"sync"
@@ -8,17 +8,17 @@ import (
 // #include <stdlib.h>
 import "C"
 
-// RefMap is a convenient way to store opaque references that can be passed to
+// refMap is a convenient way to store opaque references that can be passed to
 // C. It is useful if an API uses function pointers and you cannot pass a Go
 // pointer but only a C pointer.
-type RefMap struct {
+type refMap struct {
 	refs map[unsafe.Pointer]interface{}
 	lock sync.Mutex
 }
 
 // Put stores a value in the map. It can later be retrieved using Get. It must
 // be removed using Remove to avoid memory leaks.
-func (m *RefMap) Put(v interface{}) unsafe.Pointer {
+func (m *refMap) Put(v interface{}) unsafe.Pointer {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 	if m.refs == nil {
@@ -31,14 +31,14 @@ func (m *RefMap) Put(v interface{}) unsafe.Pointer {
 
 // Get returns a stored value previously inserted with Put. Use the same
 // reference as you got from Put.
-func (m *RefMap) Get(ref unsafe.Pointer) interface{} {
+func (m *refMap) Get(ref unsafe.Pointer) interface{} {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 	return m.refs[ref]
 }
 
 // Remove deletes a single reference from the map.
-func (m *RefMap) Remove(ref unsafe.Pointer) {
+func (m *refMap) Remove(ref unsafe.Pointer) {
 	m.lock.Lock()
 	defer m.lock.Unlock()
 	delete(m.refs, ref)

loader/loader.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+
+	"github.com/tinygo-org/tinygo/cgo"
 )
 
 // Program holds all packages and some metadata about the program as a whole.
@@ -30,11 +32,10 @@ type Program struct {
 type Package struct {
 	*Program
 	*build.Package
-	Imports    map[string]*Package
+	Imports   map[string]*Package
-	Importing  bool
+	Importing bool
-	Files      []*ast.File
+	Files     []*ast.File
-	tokenFiles map[string]*token.File
+	Pkg       *types.Package
-	Pkg        *types.Package
 	types.Info
 }
 
@@ -107,7 +108,6 @@ func (p *Program) newPackage(pkg *build.Package) *Package {
 			Scopes:     make(map[ast.Node]*types.Scope),
 			Selections: make(map[*ast.SelectorExpr]*types.Selection),
 		},
-		tokenFiles: map[string]*token.File{},
 	}
 }
 
@@ -295,16 +295,6 @@ func (p *Package) parseFiles() ([]*ast.File, error) {
 		}
 		files = append(files, f)
 	}
-	clangIncludes := ""
-	if len(p.CgoFiles) != 0 {
-		if _, err := os.Stat(filepath.Join(p.TINYGOROOT, "llvm", "tools", "clang", "lib", "Headers")); !os.IsNotExist(err) {
-			// Running from the source directory.
-			clangIncludes = filepath.Join(p.TINYGOROOT, "llvm", "tools", "clang", "lib", "Headers")
-		} else {
-			// Running from the installation directory.
-			clangIncludes = filepath.Join(p.TINYGOROOT, "lib", "clang", "include")
-		}
-	}
 	for _, file := range p.CgoFiles {
 		path := filepath.Join(p.Package.Dir, file)
 		f, err := p.parseFile(path, parser.ParseComments)
@@ -312,12 +302,22 @@ func (p *Package) parseFiles() ([]*ast.File, error) {
 			fileErrs = append(fileErrs, err)
 			continue
 		}
-		errs := p.processCgo(path, f, append(p.CFlags, "-I"+p.Package.Dir, "-I"+clangIncludes))
+		files = append(files, f)
+	}
+	if len(p.CgoFiles) != 0 {
+		clangIncludes := ""
+		if _, err := os.Stat(filepath.Join(p.TINYGOROOT, "llvm", "tools", "clang", "lib", "Headers")); !os.IsNotExist(err) {
+			// Running from the source directory.
+			clangIncludes = filepath.Join(p.TINYGOROOT, "llvm", "tools", "clang", "lib", "Headers")
+		} else {
+			// Running from the installation directory.
+			clangIncludes = filepath.Join(p.TINYGOROOT, "lib", "clang", "include")
+		}
+		generated, errs := cgo.Process(files, p.Program.Dir, p.fset, append(p.CFlags, "-I"+p.Package.Dir, "-I"+clangIncludes))
 		if errs != nil {
 			fileErrs = append(fileErrs, errs...)
-			continue
 		}
-		files = append(files, f)
+		files = append(files, generated)
 	}
 	if len(fileErrs) != 0 {
 		return nil, Errors{p, fileErrs}

testdata/cgo/extra.go

@@ -0,0 +1,6 @@
+package main
+
+// Make sure CGo supports multiple files.
+
+// int fortytwo(void);
+import "C"