From 6f63ac8d06042bcf92acd6de61ee717dcd8bad16 Mon Sep 17 00:00:00 2001 From: gedi Date: Tue, 20 Feb 2018 22:28:57 +0200 Subject: [PATCH 1/4] adapt to new compiler and linker in go tool chain --- .travis.yml | 6 +- builder.go | 2 + builder_go110.go | 331 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 builder_go110.go diff --git a/.travis.yml b/.travis.yml index 63bc304..e8904ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,18 @@ go: - 1.7.x - 1.8.x - 1.9.x + - 1.10.x go_import_path: github.com/DATA-DOG/godog install: go install github.com/DATA-DOG/godog/cmd/godog script: - - go vet ./... + - go vet github.com/DATA-DOG/godog + - go vet github.com/DATA-DOG/godog/gherkin + - go vet github.com/DATA-DOG/godog/colors - test -z "$(go fmt ./...)" # fail if not formatted properly + - godog -f progress - go test -v -race -coverprofile=coverage.txt -covermode=atomic after_success: diff --git a/builder.go b/builder.go index 0089418..d27d25f 100644 --- a/builder.go +++ b/builder.go @@ -1,3 +1,5 @@ +// +build !go1.10 + package godog import ( diff --git a/builder_go110.go b/builder_go110.go new file mode 100644 index 0000000..5857c7c --- /dev/null +++ b/builder_go110.go @@ -0,0 +1,331 @@ +// +build go1.10 + +package godog + +import ( + "bytes" + "fmt" + "go/build" + "go/parser" + "go/token" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "text/template" + "time" + "unicode" +) + +var tooldir = findToolDir() +var compiler = filepath.Join(tooldir, "compile") +var linker = filepath.Join(tooldir, "link") +var gopaths = filepath.SplitList(build.Default.GOPATH) +var goarch = build.Default.GOARCH +var goos = build.Default.GOOS + +var godogImportPath = "github.com/DATA-DOG/godog" +var runnerTemplate = template.Must(template.New("testmain").Parse(`package main + +import ( + "github.com/DATA-DOG/godog" + {{if .Contexts}}_test "{{.ImportPath}}"{{end}} + "os" +) + +func main() { + status := godog.Run("{{ .Name }}", func (suite *godog.Suite) { + os.Setenv("GODOG_TESTED_PACKAGE", "{{.ImportPath}}") + {{range .Contexts}} + _test.{{ . }}(suite) + {{end}} + }) + os.Exit(status) +}`)) + +// Build creates a test package like go test command at given target path. +// If there are no go files in tested directory, then +// it simply builds a godog executable to scan features. +// +// If there are go test files, it first builds a test +// package with standard go test command. +// +// Finally it generates godog suite executable which +// registers exported godog contexts from the test files +// of tested package. +// +// Returns the path to generated executable +func Build(bin string) error { + abs, err := filepath.Abs(".") + if err != nil { + return err + } + + // we allow package to be nil, if godog is run only when + // there is a feature file in empty directory + pkg := importPackage(abs) + src, anyContexts, err := buildTestMain(pkg) + if err != nil { + return err + } + + workdir := fmt.Sprintf(filepath.Join("%s", "godog-%d"), os.TempDir(), time.Now().UnixNano()) + testdir := workdir + + // if none of test files exist, or there are no contexts found + // we will skip test package compilation, since it is useless + if anyContexts { + // first of all compile test package dependencies + // that will save us many compilations for dependencies + // go does it better + out, err := exec.Command("go", "test", "-i").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to compile package: %s, reason: %v, output: %s", pkg.Name, err, string(out)) + } + + // builds and compile the tested package. + // generated test executable will be removed + // since we do not need it for godog suite. + // we also print back the temp WORK directory + // go has built. We will reuse it for our suite workdir. + out, err = exec.Command("go", "test", "-c", "-work", "-o", "/dev/null").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to compile tested package: %s, reason: %v, output: %s", pkg.Name, err, string(out)) + } + + // extract go-build temporary directory as our workdir + workdir = strings.TrimSpace(string(out)) + if !strings.HasPrefix(workdir, "WORK=") { + return fmt.Errorf("expected WORK dir path, but got: %s", workdir) + } + workdir = strings.Replace(workdir, "WORK=", "", 1) + testdir = filepath.Join(workdir, "b001") + } else { + // still need to create temporary workdir + if err = os.MkdirAll(testdir, 0755); err != nil { + return err + } + } + defer os.RemoveAll(workdir) + + // replace _testmain.go file with our own + testmain := filepath.Join(testdir, "_testmain.go") + err = ioutil.WriteFile(testmain, src, 0644) + if err != nil { + return err + } + + // godog library may not be imported in tested package + // but we need it for our testmain package. + // So we look it up in available source paths + // including vendor directory, supported since 1.5. + try := maybeVendorPaths(abs) + for _, d := range build.Default.SrcDirs() { + try = append(try, filepath.Join(d, godogImportPath)) + } + godogPkg, err := locatePackage(try) + if err != nil { + return err + } + + // make sure godog package archive is installed, gherkin + // will be installed as dependency of godog + cmd := exec.Command("go", "install", godogPkg.ImportPath) + cmd.Env = os.Environ() + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to install godog package: %s, reason: %v", string(out), err) + } + + // collect all possible package dirs, will be + // used for includes and linker + pkgDirs := []string{workdir, testdir} + for _, gopath := range gopaths { + pkgDirs = append(pkgDirs, filepath.Join(gopath, "pkg", goos+"_"+goarch)) + } + pkgDirs = uniqStringList(pkgDirs) + + // compile godog testmain package archive + // we do not depend on CGO so a lot of checks are not necessary + testMainPkgOut := filepath.Join(testdir, "main.a") + args := []string{ + "-o", testMainPkgOut, + // "-trimpath", workdir, + "-importcfg", filepath.Join(testdir, "importcfg.link"), + "-p", "main", + "-complete", + } + // if godog library is in vendor directory + // link it with import map + if i := strings.LastIndex(godogPkg.ImportPath, "vendor/"); i != -1 { + args = append(args, "-importmap", godogImportPath+"="+godogPkg.ImportPath) + } + for _, inc := range pkgDirs { + args = append(args, "-I", inc) + } + args = append(args, "-pack", testmain) + cmd = exec.Command(compiler, args...) + cmd.Env = os.Environ() + out, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to compile testmain package: %v - output: %s", err, string(out)) + } + + // link test suite executable + args = []string{ + "-o", bin, + "-importcfg", filepath.Join(testdir, "importcfg.link"), + "-buildmode=exe", + } + for _, link := range pkgDirs { + args = append(args, "-L", link) + } + args = append(args, testMainPkgOut) + cmd = exec.Command(linker, args...) + cmd.Env = os.Environ() + + out, err = cmd.CombinedOutput() + if err != nil { + msg := `failed to link test executable: + reason: %s + command: %s` + return fmt.Errorf(msg, string(out), linker+" '"+strings.Join(args, "' '")+"'") + } + + return nil +} + +func locatePackage(try []string) (*build.Package, error) { + for _, p := range try { + abs, err := filepath.Abs(p) + if err != nil { + continue + } + pkg, err := build.ImportDir(abs, 0) + if err != nil { + continue + } + return pkg, nil + } + return nil, fmt.Errorf("failed to find godog package in any of:\n%s", strings.Join(try, "\n")) +} + +func importPackage(dir string) *build.Package { + pkg, _ := build.ImportDir(dir, 0) + + // normalize import path for local import packages + // taken from go source code + // see: https://github.com/golang/go/blob/go1.7rc5/src/cmd/go/pkg.go#L279 + if pkg != nil && pkg.ImportPath == "." { + pkg.ImportPath = path.Join("_", strings.Map(makeImportValid, filepath.ToSlash(dir))) + } + + return pkg +} + +// from go src +func makeImportValid(r rune) rune { + // Should match Go spec, compilers, and ../../go/parser/parser.go:/isValidImport. + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { + return '_' + } + return r +} + +func uniqStringList(strs []string) (unique []string) { + uniq := make(map[string]void, len(strs)) + for _, s := range strs { + if _, ok := uniq[s]; !ok { + uniq[s] = void{} + unique = append(unique, s) + } + } + return +} + +// buildTestMain if given package is valid +// it scans test files for contexts +// and produces a testmain source code. +func buildTestMain(pkg *build.Package) ([]byte, bool, error) { + var contexts []string + var importPath string + name := "main" + if nil != pkg { + ctxs, err := processPackageTestFiles( + pkg.TestGoFiles, + pkg.XTestGoFiles, + ) + if err != nil { + return nil, false, err + } + contexts = ctxs + importPath = pkg.ImportPath + name = pkg.Name + } + + data := struct { + Name string + Contexts []string + ImportPath string + }{name, contexts, importPath} + + var buf bytes.Buffer + if err := runnerTemplate.Execute(&buf, data); err != nil { + return nil, len(contexts) > 0, err + } + return buf.Bytes(), len(contexts) > 0, nil +} + +// maybeVendorPaths determines possible vendor paths +// which goes levels down from given directory +// until it reaches GOPATH source dir +func maybeVendorPaths(dir string) (paths []string) { + for _, gopath := range gopaths { + gopath = filepath.Join(gopath, "src") + for strings.HasPrefix(dir, gopath) && dir != gopath { + paths = append(paths, filepath.Join(dir, "vendor", godogImportPath)) + dir = filepath.Dir(dir) + } + } + return +} + +// processPackageTestFiles runs through ast of each test +// file pack and looks for godog suite contexts to register +// on run +func processPackageTestFiles(packs ...[]string) ([]string, error) { + var ctxs []string + fset := token.NewFileSet() + for _, pack := range packs { + for _, testFile := range pack { + node, err := parser.ParseFile(fset, testFile, nil, 0) + if err != nil { + return ctxs, err + } + + ctxs = append(ctxs, astContexts(node)...) + } + } + var failed []string + for _, ctx := range ctxs { + runes := []rune(ctx) + if unicode.IsLower(runes[0]) { + expected := append([]rune{unicode.ToUpper(runes[0])}, runes[1:]...) + failed = append(failed, fmt.Sprintf("%s - should be: %s", ctx, string(expected))) + } + } + if len(failed) > 0 { + return ctxs, fmt.Errorf("godog contexts must be exported:\n\t%s", strings.Join(failed, "\n\t")) + } + return ctxs, nil +} + +func findToolDir() string { + if out, err := exec.Command("go", "env", "GOTOOLDIR").Output(); err != nil { + return filepath.Clean(strings.TrimSpace(string(out))) + } + return filepath.Clean(build.ToolDir) +} From 3456668e83092ea3bf457c0901bb19ab0b68bb6e Mon Sep 17 00:00:00 2001 From: gedi Date: Sat, 3 Mar 2018 20:13:24 +0200 Subject: [PATCH 2/4] builds the package dependency tree when there are no go sources --- builder_go110.go | 123 ++++++++++++++++++++++++++++++----------------- builder_test.go | 38 +++++---------- 2 files changed, 92 insertions(+), 69 deletions(-) diff --git a/builder_go110.go b/builder_go110.go index 5857c7c..c8dee05 100644 --- a/builder_go110.go +++ b/builder_go110.go @@ -24,6 +24,7 @@ var compiler = filepath.Join(tooldir, "compile") var linker = filepath.Join(tooldir, "link") var gopaths = filepath.SplitList(build.Default.GOPATH) var goarch = build.Default.GOARCH +var goroot = build.Default.GOROOT var goos = build.Default.GOOS var godogImportPath = "github.com/DATA-DOG/godog" @@ -121,11 +122,7 @@ func Build(bin string) error { // but we need it for our testmain package. // So we look it up in available source paths // including vendor directory, supported since 1.5. - try := maybeVendorPaths(abs) - for _, d := range build.Default.SrcDirs() { - try = append(try, filepath.Join(d, godogImportPath)) - } - godogPkg, err := locatePackage(try) + godogPkg, err := locatePackage(godogImportPath) if err != nil { return err } @@ -139,32 +136,48 @@ func Build(bin string) error { return fmt.Errorf("failed to install godog package: %s, reason: %v", string(out), err) } - // collect all possible package dirs, will be - // used for includes and linker - pkgDirs := []string{workdir, testdir} - for _, gopath := range gopaths { - pkgDirs = append(pkgDirs, filepath.Join(gopath, "pkg", goos+"_"+goarch)) - } - pkgDirs = uniqStringList(pkgDirs) - // compile godog testmain package archive // we do not depend on CGO so a lot of checks are not necessary testMainPkgOut := filepath.Join(testdir, "main.a") args := []string{ "-o", testMainPkgOut, - // "-trimpath", workdir, - "-importcfg", filepath.Join(testdir, "importcfg.link"), "-p", "main", "-complete", } - // if godog library is in vendor directory - // link it with import map - if i := strings.LastIndex(godogPkg.ImportPath, "vendor/"); i != -1 { - args = append(args, "-importmap", godogImportPath+"="+godogPkg.ImportPath) - } - for _, inc := range pkgDirs { - args = append(args, "-I", inc) + + var in *os.File + cfg := filepath.Join(testdir, "importcfg.link") + args = append(args, "-importcfg", cfg) + if _, err := os.Stat(cfg); err == nil { + // there were go sources + in, err = os.OpenFile(cfg, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return err + } + } else { + // there were no go sources in the directory + // so we need to build all dependency tree ourselves + in, err = os.Create(cfg) + if err != nil { + return err + } + fmt.Fprintln(in, "# import config") + + deps := make(map[string]string) + if err := dependencies(godogPkg, deps); err != nil { + return err + } + + for pkgName, pkgObj := range deps { + if i := strings.LastIndex(pkgName, "vendor/"); i != -1 { + fmt.Fprintf(in, "importmap %s=%s\n", pkgName, pkgObj) + } else { + fmt.Fprintf(in, "packagefile %s=%s\n", pkgName, pkgObj) + } + } } + in.Close() + args = append(args, "-pack", testmain) cmd = exec.Command(compiler, args...) cmd.Env = os.Environ() @@ -176,12 +189,9 @@ func Build(bin string) error { // link test suite executable args = []string{ "-o", bin, - "-importcfg", filepath.Join(testdir, "importcfg.link"), + "-importcfg", cfg, "-buildmode=exe", } - for _, link := range pkgDirs { - args = append(args, "-L", link) - } args = append(args, testMainPkgOut) cmd = exec.Command(linker, args...) cmd.Env = os.Environ() @@ -197,9 +207,9 @@ func Build(bin string) error { return nil } -func locatePackage(try []string) (*build.Package, error) { - for _, p := range try { - abs, err := filepath.Abs(p) +func locatePackage(name string) (*build.Package, error) { + for _, p := range build.Default.SrcDirs() { + abs, err := filepath.Abs(filepath.Join(p, name)) if err != nil { continue } @@ -209,7 +219,26 @@ func locatePackage(try []string) (*build.Package, error) { } return pkg, nil } - return nil, fmt.Errorf("failed to find godog package in any of:\n%s", strings.Join(try, "\n")) + + // search vendor paths + dir, err := filepath.Abs(".") + if err != nil { + return nil, err + } + + for _, gopath := range gopaths { + gopath = filepath.Join(gopath, "src") + for strings.HasPrefix(dir, gopath) && dir != gopath { + pkg, err := build.ImportDir(filepath.Join(dir, "vendor", name), 0) + if err != nil { + dir = filepath.Dir(dir) + continue + } + return pkg, nil + } + } + + return nil, fmt.Errorf("failed to find %s package in any of:\n%s", name, strings.Join(build.Default.SrcDirs(), "\n")) } func importPackage(dir string) *build.Package { @@ -279,20 +308,6 @@ func buildTestMain(pkg *build.Package) ([]byte, bool, error) { return buf.Bytes(), len(contexts) > 0, nil } -// maybeVendorPaths determines possible vendor paths -// which goes levels down from given directory -// until it reaches GOPATH source dir -func maybeVendorPaths(dir string) (paths []string) { - for _, gopath := range gopaths { - gopath = filepath.Join(gopath, "src") - for strings.HasPrefix(dir, gopath) && dir != gopath { - paths = append(paths, filepath.Join(dir, "vendor", godogImportPath)) - dir = filepath.Dir(dir) - } - } - return -} - // processPackageTestFiles runs through ast of each test // file pack and looks for godog suite contexts to register // on run @@ -329,3 +344,23 @@ func findToolDir() string { } return filepath.Clean(build.ToolDir) } + +func dependencies(pkg *build.Package, visited map[string]string) error { + visited[pkg.ImportPath] = pkg.PkgObj + for _, name := range pkg.Imports { + if _, ok := visited[name]; ok { + continue + } + + next, err := locatePackage(name) + if err != nil { + return err + } + + visited[name] = pkg.PkgObj + if err := dependencies(next, visited); err != nil { + return err + } + } + return nil +} diff --git a/builder_test.go b/builder_test.go index a5bc8b6..83d95de 100644 --- a/builder_test.go +++ b/builder_test.go @@ -1,40 +1,28 @@ package godog import ( - "go/build" "os" "path/filepath" - "reflect" "testing" ) -func TestVendorPaths(t *testing.T) { - gopaths = []string{"/go"} +func TestDeps(t *testing.T) { + t.Log("hh") - type Case struct { - dir string - expect []string + abs, err := filepath.Abs(".") + if err != nil { + t.Fatal(err) } - cases := []Case{ - {"/go", []string{}}, - {"/go/src", []string{}}, - {"/go/src/project", []string{"/go/src/project/vendor"}}, - {"/go/src/party/project", []string{"/go/src/party/project/vendor", "/go/src/party/vendor"}}, + // we allow package to be nil, if godog is run only when + // there is a feature file in empty directory + pkg := importPackage(abs) + deps := make(map[string]string) + err = dependencies(pkg, deps) + if err != nil { + t.Fatal(err) } - - for i, c := range cases { - actual := maybeVendorPaths(c.dir) - var expect []string - for _, s := range c.expect { - expect = append(expect, filepath.Join(s, godogImportPath)) - } - if !reflect.DeepEqual(expect, actual) { - t.Fatalf("case %d expected %+v, got %+v", i, expect, actual) - } - } - - gopaths = filepath.SplitList(build.Default.GOPATH) + t.Log(deps) } func TestBuildTestRunner(t *testing.T) { From f9f6c208d62980f1511abfd5724e5764cd4e68a4 Mon Sep 17 00:00:00 2001 From: gedi Date: Sat, 3 Mar 2018 20:48:45 +0200 Subject: [PATCH 3/4] vendor package support missing still for go1.10 --- builder_go110.go | 6 +++--- builder_test.go | 19 ------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/builder_go110.go b/builder_go110.go index c8dee05..568eede 100644 --- a/builder_go110.go +++ b/builder_go110.go @@ -170,10 +170,10 @@ func Build(bin string) error { for pkgName, pkgObj := range deps { if i := strings.LastIndex(pkgName, "vendor/"); i != -1 { - fmt.Fprintf(in, "importmap %s=%s\n", pkgName, pkgObj) - } else { - fmt.Fprintf(in, "packagefile %s=%s\n", pkgName, pkgObj) + name := pkgName[i+7:] + fmt.Fprintf(in, "importmap %s=%s\n", name, pkgName) } + fmt.Fprintf(in, "packagefile %s=%s\n", pkgName, pkgObj) } } in.Close() diff --git a/builder_test.go b/builder_test.go index 83d95de..4a1c5ac 100644 --- a/builder_test.go +++ b/builder_test.go @@ -6,25 +6,6 @@ import ( "testing" ) -func TestDeps(t *testing.T) { - t.Log("hh") - - abs, err := filepath.Abs(".") - if err != nil { - t.Fatal(err) - } - - // we allow package to be nil, if godog is run only when - // there is a feature file in empty directory - pkg := importPackage(abs) - deps := make(map[string]string) - err = dependencies(pkg, deps) - if err != nil { - t.Fatal(err) - } - t.Log(deps) -} - func TestBuildTestRunner(t *testing.T) { bin := filepath.Join(os.TempDir(), "godog.test") if err := Build(bin); err != nil { From bf43aba2d8398082f46203ff56e9f724d8eacb8d Mon Sep 17 00:00:00 2001 From: gedi Date: Sun, 4 Mar 2018 09:56:22 +0200 Subject: [PATCH 4/4] support godog package in vendor directory --- builder_go110.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/builder_go110.go b/builder_go110.go index 568eede..49f49d7 100644 --- a/builder_go110.go +++ b/builder_go110.go @@ -129,7 +129,7 @@ func Build(bin string) error { // make sure godog package archive is installed, gherkin // will be installed as dependency of godog - cmd := exec.Command("go", "install", godogPkg.ImportPath) + cmd := exec.Command("go", "install", "-i", godogPkg.ImportPath) cmd.Env = os.Environ() out, err := cmd.CombinedOutput() if err != nil { @@ -196,6 +196,26 @@ func Build(bin string) error { cmd = exec.Command(linker, args...) cmd.Env = os.Environ() + // in case if build is without contexts, need to remove import maps + if testdir == workdir { + data, err := ioutil.ReadFile(cfg) + if err != nil { + return err + } + + lines := strings.Split(string(data), "\n") + var fixed []string + for _, line := range lines { + if strings.Index(line, "importmap") == 0 { + continue + } + fixed = append(fixed, line) + } + if err := ioutil.WriteFile(cfg, []byte(strings.Join(fixed, "\n")), 0600); err != nil { + return err + } + } + out, err = cmd.CombinedOutput() if err != nil { msg := `failed to link test executable: