2019-04-23 10:34:28 +02:00
|
|
|
package packages
|
|
|
|
|
|
|
|
import (
|
2019-06-22 23:51:58 +02:00
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
2019-04-23 10:34:28 +02:00
|
|
|
"go/parser"
|
|
|
|
"go/token"
|
2020-04-07 09:29:05 +02:00
|
|
|
"os"
|
2019-04-23 10:34:28 +02:00
|
|
|
"path/filepath"
|
2020-04-07 09:29:05 +02:00
|
|
|
"sort"
|
2019-04-23 10:34:28 +02:00
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
)
|
|
|
|
|
|
|
|
// processGolistOverlay provides rudimentary support for adding
|
|
|
|
// files that don't exist on disk to an overlay. The results can be
|
|
|
|
// sometimes incorrect.
|
|
|
|
// TODO(matloob): Handle unsupported cases, including the following:
|
|
|
|
// - determining the correct package to add given a new import path
|
2020-04-07 09:29:05 +02:00
|
|
|
func (state *golistState) processGolistOverlay(response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) {
|
2019-04-23 10:34:28 +02:00
|
|
|
havePkgs := make(map[string]string) // importPath -> non-test package ID
|
|
|
|
needPkgsSet := make(map[string]bool)
|
|
|
|
modifiedPkgsSet := make(map[string]bool)
|
|
|
|
|
2019-06-22 23:51:58 +02:00
|
|
|
for _, pkg := range response.dr.Packages {
|
2019-04-23 10:34:28 +02:00
|
|
|
// This is an approximation of import path to id. This can be
|
|
|
|
// wrong for tests, vendored packages, and a number of other cases.
|
|
|
|
havePkgs[pkg.PkgPath] = pkg.ID
|
|
|
|
}
|
|
|
|
|
2019-06-28 09:01:50 +02:00
|
|
|
// If no new imports are added, it is safe to avoid loading any needPkgs.
|
|
|
|
// Otherwise, it's hard to tell which package is actually being loaded
|
|
|
|
// (due to vendoring) and whether any modified package will show up
|
|
|
|
// in the transitive set of dependencies (because new imports are added,
|
|
|
|
// potentially modifying the transitive set of dependencies).
|
|
|
|
var overlayAddsImports bool
|
|
|
|
|
2020-04-07 09:29:05 +02:00
|
|
|
// If both a package and its test package are created by the overlay, we
|
|
|
|
// need the real package first. Process all non-test files before test
|
|
|
|
// files, and make the whole process deterministic while we're at it.
|
|
|
|
var overlayFiles []string
|
|
|
|
for opath := range state.cfg.Overlay {
|
|
|
|
overlayFiles = append(overlayFiles, opath)
|
|
|
|
}
|
|
|
|
sort.Slice(overlayFiles, func(i, j int) bool {
|
|
|
|
iTest := strings.HasSuffix(overlayFiles[i], "_test.go")
|
|
|
|
jTest := strings.HasSuffix(overlayFiles[j], "_test.go")
|
|
|
|
if iTest != jTest {
|
|
|
|
return !iTest // non-tests are before tests.
|
|
|
|
}
|
|
|
|
return overlayFiles[i] < overlayFiles[j]
|
|
|
|
})
|
|
|
|
for _, opath := range overlayFiles {
|
|
|
|
contents := state.cfg.Overlay[opath]
|
2019-06-22 23:51:58 +02:00
|
|
|
base := filepath.Base(opath)
|
|
|
|
dir := filepath.Dir(opath)
|
2019-10-16 22:52:29 +02:00
|
|
|
var pkg *Package // if opath belongs to both a package and its test variant, this will be the test variant
|
2019-06-22 23:51:58 +02:00
|
|
|
var testVariantOf *Package // if opath is a test file, this is the package it is testing
|
|
|
|
var fileExists bool
|
2019-10-16 22:52:29 +02:00
|
|
|
isTestFile := strings.HasSuffix(opath, "_test.go")
|
2019-06-22 23:51:58 +02:00
|
|
|
pkgName, ok := extractPackageName(opath, contents)
|
|
|
|
if !ok {
|
|
|
|
// Don't bother adding a file that doesn't even have a parsable package statement
|
|
|
|
// to the overlay.
|
2019-04-23 10:34:28 +02:00
|
|
|
continue
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
nextPackage:
|
|
|
|
for _, p := range response.dr.Packages {
|
2019-10-16 22:52:29 +02:00
|
|
|
if pkgName != p.Name && p.ID != "command-line-arguments" {
|
2019-06-22 23:51:58 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, f := range p.GoFiles {
|
|
|
|
if !sameFile(filepath.Dir(f), dir) {
|
|
|
|
continue
|
2019-04-23 10:34:28 +02:00
|
|
|
}
|
2019-10-16 22:52:29 +02:00
|
|
|
// Make sure to capture information on the package's test variant, if needed.
|
|
|
|
if isTestFile && !hasTestFiles(p) {
|
2019-06-22 23:51:58 +02:00
|
|
|
// TODO(matloob): Are there packages other than the 'production' variant
|
|
|
|
// of a package that this can match? This shouldn't match the test main package
|
|
|
|
// because the file is generated in another directory.
|
|
|
|
testVariantOf = p
|
|
|
|
continue nextPackage
|
|
|
|
}
|
2020-04-07 09:29:05 +02:00
|
|
|
// We must have already seen the package of which this is a test variant.
|
2019-10-16 22:52:29 +02:00
|
|
|
if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath {
|
|
|
|
if hasTestFiles(p) {
|
|
|
|
testVariantOf = pkg
|
|
|
|
}
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
pkg = p
|
2019-04-23 10:34:28 +02:00
|
|
|
if filepath.Base(f) == base {
|
|
|
|
fileExists = true
|
|
|
|
}
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
// The overlay could have included an entirely new package.
|
|
|
|
if pkg == nil {
|
|
|
|
// Try to find the module or gopath dir the file is contained in.
|
|
|
|
// Then for modules, add the module opath to the beginning.
|
2020-04-07 09:29:05 +02:00
|
|
|
pkgPath, ok, err := state.getPkgPath(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
2019-10-23 23:11:40 +02:00
|
|
|
if !ok {
|
2019-06-22 23:51:58 +02:00
|
|
|
break
|
|
|
|
}
|
|
|
|
isXTest := strings.HasSuffix(pkgName, "_test")
|
|
|
|
if isXTest {
|
|
|
|
pkgPath += "_test"
|
|
|
|
}
|
|
|
|
id := pkgPath
|
2019-10-16 22:52:29 +02:00
|
|
|
if isTestFile && !isXTest {
|
2019-06-22 23:51:58 +02:00
|
|
|
id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
|
|
|
|
}
|
|
|
|
// Try to reclaim a package with the same id if it exists in the response.
|
|
|
|
for _, p := range response.dr.Packages {
|
|
|
|
if reclaimPackage(p, id, opath, contents) {
|
|
|
|
pkg = p
|
|
|
|
break
|
2019-04-23 10:34:28 +02:00
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
// Otherwise, create a new package
|
|
|
|
if pkg == nil {
|
|
|
|
pkg = &Package{PkgPath: pkgPath, ID: id, Name: pkgName, Imports: make(map[string]*Package)}
|
|
|
|
response.addPackage(pkg)
|
|
|
|
havePkgs[pkg.PkgPath] = id
|
|
|
|
// Add the production package's sources for a test variant.
|
2019-10-16 22:52:29 +02:00
|
|
|
if isTestFile && !isXTest && testVariantOf != nil {
|
2019-06-22 23:51:58 +02:00
|
|
|
pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
|
|
|
|
pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
|
2020-04-07 09:29:05 +02:00
|
|
|
// Add the package under test and its imports to the test variant.
|
|
|
|
pkg.forTest = testVariantOf.PkgPath
|
|
|
|
for k, v := range testVariantOf.Imports {
|
|
|
|
pkg.Imports[k] = &Package{ID: v.ID}
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !fileExists {
|
|
|
|
pkg.GoFiles = append(pkg.GoFiles, opath)
|
|
|
|
// TODO(matloob): Adding the file to CompiledGoFiles can exhibit the wrong behavior
|
|
|
|
// if the file will be ignored due to its build tags.
|
|
|
|
pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, opath)
|
|
|
|
modifiedPkgsSet[pkg.ID] = true
|
|
|
|
}
|
|
|
|
imports, err := extractImports(opath, contents)
|
|
|
|
if err != nil {
|
|
|
|
// Let the parser or type checker report errors later.
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
for _, imp := range imports {
|
2020-04-07 09:29:05 +02:00
|
|
|
if _, found := pkg.Imports[imp]; found {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
overlayAddsImports = true
|
|
|
|
id, ok := havePkgs[imp]
|
|
|
|
if !ok {
|
|
|
|
var err error
|
|
|
|
id, err = state.resolveImport(dir, imp)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
2019-10-16 22:52:29 +02:00
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
2020-04-07 09:29:05 +02:00
|
|
|
pkg.Imports[imp] = &Package{ID: id}
|
|
|
|
// Add dependencies to the non-test variant version of this package as well.
|
|
|
|
if testVariantOf != nil {
|
|
|
|
testVariantOf.Imports[imp] = &Package{ID: id}
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-07 09:29:05 +02:00
|
|
|
// toPkgPath guesses the package path given the id.
|
|
|
|
toPkgPath := func(sourceDir, id string) (string, error) {
|
|
|
|
if i := strings.IndexByte(id, ' '); i >= 0 {
|
|
|
|
return state.resolveImport(sourceDir, id[:i])
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
2020-04-07 09:29:05 +02:00
|
|
|
return state.resolveImport(sourceDir, id)
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
|
2020-04-07 09:29:05 +02:00
|
|
|
// Now that new packages have been created, do another pass to determine
|
|
|
|
// the new set of missing packages.
|
2019-06-22 23:51:58 +02:00
|
|
|
for _, pkg := range response.dr.Packages {
|
|
|
|
for _, imp := range pkg.Imports {
|
2020-04-07 09:29:05 +02:00
|
|
|
if len(pkg.GoFiles) == 0 {
|
|
|
|
return nil, nil, fmt.Errorf("cannot resolve imports for package %q with no Go files", pkg.PkgPath)
|
|
|
|
}
|
|
|
|
pkgPath, err := toPkgPath(filepath.Dir(pkg.GoFiles[0]), imp.ID)
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
if _, ok := havePkgs[pkgPath]; !ok {
|
|
|
|
needPkgsSet[pkgPath] = true
|
2019-04-23 10:34:28 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-28 09:01:50 +02:00
|
|
|
if overlayAddsImports {
|
|
|
|
needPkgs = make([]string, 0, len(needPkgsSet))
|
|
|
|
for pkg := range needPkgsSet {
|
|
|
|
needPkgs = append(needPkgs, pkg)
|
|
|
|
}
|
2019-04-23 10:34:28 +02:00
|
|
|
}
|
|
|
|
modifiedPkgs = make([]string, 0, len(modifiedPkgsSet))
|
|
|
|
for pkg := range modifiedPkgsSet {
|
|
|
|
modifiedPkgs = append(modifiedPkgs, pkg)
|
|
|
|
}
|
|
|
|
return modifiedPkgs, needPkgs, err
|
|
|
|
}
|
|
|
|
|
2020-04-07 09:29:05 +02:00
|
|
|
// resolveImport finds the the ID of a package given its import path.
|
|
|
|
// In particular, it will find the right vendored copy when in GOPATH mode.
|
|
|
|
func (state *golistState) resolveImport(sourceDir, importPath string) (string, error) {
|
|
|
|
env, err := state.getEnv()
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
if env["GOMOD"] != "" {
|
|
|
|
return importPath, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
searchDir := sourceDir
|
|
|
|
for {
|
|
|
|
vendorDir := filepath.Join(searchDir, "vendor")
|
|
|
|
exists, ok := state.vendorDirs[vendorDir]
|
|
|
|
if !ok {
|
|
|
|
info, err := os.Stat(vendorDir)
|
|
|
|
exists = err == nil && info.IsDir()
|
|
|
|
state.vendorDirs[vendorDir] = exists
|
|
|
|
}
|
|
|
|
|
|
|
|
if exists {
|
|
|
|
vendoredPath := filepath.Join(vendorDir, importPath)
|
|
|
|
if info, err := os.Stat(vendoredPath); err == nil && info.IsDir() {
|
|
|
|
// We should probably check for .go files here, but shame on anyone who fools us.
|
|
|
|
path, ok, err := state.getPkgPath(vendoredPath)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
if ok {
|
|
|
|
return path, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// We know we've hit the top of the filesystem when we Dir / and get /,
|
|
|
|
// or C:\ and get C:\, etc.
|
|
|
|
next := filepath.Dir(searchDir)
|
|
|
|
if next == searchDir {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
searchDir = next
|
|
|
|
}
|
|
|
|
return importPath, nil
|
|
|
|
}
|
|
|
|
|
2019-06-22 23:51:58 +02:00
|
|
|
func hasTestFiles(p *Package) bool {
|
|
|
|
for _, f := range p.GoFiles {
|
|
|
|
if strings.HasSuffix(f, "_test.go") {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2020-04-07 09:29:05 +02:00
|
|
|
// determineRootDirs returns a mapping from absolute directories that could
|
|
|
|
// contain code to their corresponding import path prefixes.
|
|
|
|
func (state *golistState) determineRootDirs() (map[string]string, error) {
|
|
|
|
env, err := state.getEnv()
|
2019-06-22 23:51:58 +02:00
|
|
|
if err != nil {
|
2020-04-07 09:29:05 +02:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if env["GOMOD"] != "" {
|
|
|
|
state.rootsOnce.Do(func() {
|
|
|
|
state.rootDirs, state.rootDirsError = state.determineRootDirsModules()
|
|
|
|
})
|
|
|
|
} else {
|
|
|
|
state.rootsOnce.Do(func() {
|
|
|
|
state.rootDirs, state.rootDirsError = state.determineRootDirsGOPATH()
|
|
|
|
})
|
|
|
|
}
|
|
|
|
return state.rootDirs, state.rootDirsError
|
|
|
|
}
|
|
|
|
|
|
|
|
func (state *golistState) determineRootDirsModules() (map[string]string, error) {
|
2020-04-12 19:29:24 +02:00
|
|
|
// This will only return the root directory for the main module.
|
|
|
|
// For now we only support overlays in main modules.
|
|
|
|
// Editing files in the module cache isn't a great idea, so we don't
|
|
|
|
// plan to ever support that, but editing files in replaced modules
|
|
|
|
// is something we may want to support. To do that, we'll want to
|
|
|
|
// do a go list -m to determine the replaced module's module path and
|
|
|
|
// directory, and then a go list -m {{with .Replace}}{{.Dir}}{{end}} <replaced module's path>
|
|
|
|
// from the main module to determine if that module is actually a replacement.
|
|
|
|
// See bcmills's comment here: https://github.com/golang/go/issues/37629#issuecomment-594179751
|
|
|
|
// for more information.
|
|
|
|
out, err := state.invokeGo("list", "-m", "-json")
|
2020-04-07 09:29:05 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
m := map[string]string{}
|
|
|
|
type jsonMod struct{ Path, Dir string }
|
|
|
|
for dec := json.NewDecoder(out); dec.More(); {
|
|
|
|
mod := new(jsonMod)
|
|
|
|
if err := dec.Decode(mod); err != nil {
|
2020-04-07 09:29:05 +02:00
|
|
|
return nil, err
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
if mod.Dir != "" && mod.Path != "" {
|
|
|
|
// This is a valid module; add it to the map.
|
2020-04-07 09:29:05 +02:00
|
|
|
absDir, err := filepath.Abs(mod.Dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
m[absDir] = mod.Path
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
}
|
2020-04-07 09:29:05 +02:00
|
|
|
return m, nil
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
|
2020-04-07 09:29:05 +02:00
|
|
|
func (state *golistState) determineRootDirsGOPATH() (map[string]string, error) {
|
2019-06-22 23:51:58 +02:00
|
|
|
m := map[string]string{}
|
2020-04-07 09:29:05 +02:00
|
|
|
for _, dir := range filepath.SplitList(state.mustGetEnv()["GOPATH"]) {
|
|
|
|
absDir, err := filepath.Abs(dir)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
m[filepath.Join(absDir, "src")] = ""
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
2020-04-07 09:29:05 +02:00
|
|
|
return m, nil
|
2019-06-22 23:51:58 +02:00
|
|
|
}
|
|
|
|
|
2019-04-23 10:34:28 +02:00
|
|
|
func extractImports(filename string, contents []byte) ([]string, error) {
|
|
|
|
f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset?
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
var res []string
|
|
|
|
for _, imp := range f.Imports {
|
|
|
|
quotedPath := imp.Path.Value
|
|
|
|
path, err := strconv.Unquote(quotedPath)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
res = append(res, path)
|
|
|
|
}
|
|
|
|
return res, nil
|
|
|
|
}
|
|
|
|
|
2019-06-22 23:51:58 +02:00
|
|
|
// reclaimPackage attempts to reuse a package that failed to load in an overlay.
|
2019-05-07 21:42:24 +02:00
|
|
|
//
|
|
|
|
// If the package has errors and has no Name, GoFiles, or Imports,
|
|
|
|
// then it's possible that it doesn't yet exist on disk.
|
2019-06-22 23:51:58 +02:00
|
|
|
func reclaimPackage(pkg *Package, id string, filename string, contents []byte) bool {
|
2019-04-23 10:34:28 +02:00
|
|
|
// TODO(rstambler): Check the message of the actual error?
|
|
|
|
// It differs between $GOPATH and module mode.
|
2019-06-22 23:51:58 +02:00
|
|
|
if pkg.ID != id {
|
|
|
|
return false
|
|
|
|
}
|
2019-04-23 10:34:28 +02:00
|
|
|
if len(pkg.Errors) != 1 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if pkg.Name != "" || pkg.ExportFile != "" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if len(pkg.GoFiles) > 0 || len(pkg.CompiledGoFiles) > 0 || len(pkg.OtherFiles) > 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if len(pkg.Imports) > 0 {
|
|
|
|
return false
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
pkgName, ok := extractPackageName(filename, contents)
|
|
|
|
if !ok {
|
2019-04-23 10:34:28 +02:00
|
|
|
return false
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
pkg.Name = pkgName
|
2019-05-07 21:42:24 +02:00
|
|
|
pkg.Errors = nil
|
2019-04-23 10:34:28 +02:00
|
|
|
return true
|
|
|
|
}
|
2019-06-22 23:51:58 +02:00
|
|
|
|
|
|
|
func extractPackageName(filename string, contents []byte) (string, bool) {
|
|
|
|
// TODO(rstambler): Check the message of the actual error?
|
|
|
|
// It differs between $GOPATH and module mode.
|
|
|
|
f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.PackageClauseOnly) // TODO(matloob): reuse fileset?
|
|
|
|
if err != nil {
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
return f.Name.Name, true
|
|
|
|
}
|