dirs.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package doc
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"log"
    11  	"os"
    12  	"os/exec"
    13  	"path/filepath"
    14  	"regexp"
    15  	"strings"
    16  	"sync"
    17  
    18  	"cmd/go/internal/cfg"
    19  	"cmd/go/internal/modload"
    20  
    21  	"golang.org/x/mod/semver"
    22  )
    23  
    24  // A Dir describes a directory holding code by specifying
    25  // the expected import path and the file system directory.
    26  type Dir struct {
    27  	importPath string // import path for that dir
    28  	dir        string // file system directory
    29  	inModule   bool
    30  }
    31  
    32  // Dirs is a structure for scanning the directory tree.
    33  // Its Next method returns the next Go source directory it finds.
    34  // Although it can be used to scan the tree multiple times, it
    35  // only walks the tree once, caching the data it finds.
    36  type Dirs struct {
    37  	scan   chan Dir // Directories generated by walk.
    38  	hist   []Dir    // History of reported Dirs.
    39  	offset int      // Counter for Next.
    40  }
    41  
    42  var dirs Dirs
    43  
    44  // dirsInit starts the scanning of package directories in GOROOT and GOPATH. Any
    45  // extra paths passed to it are included in the channel.
    46  func dirsInit(extra ...Dir) {
    47  	dirs.hist = make([]Dir, 0, 1000)
    48  	dirs.hist = append(dirs.hist, extra...)
    49  	dirs.scan = make(chan Dir)
    50  	go dirs.walk(codeRoots())
    51  }
    52  
    53  // goCmd returns the "go" command path corresponding to cfg.GOROOT.
    54  func goCmd() string {
    55  	if cfg.GOROOT == "" {
    56  		return "go"
    57  	}
    58  	return filepath.Join(cfg.GOROOT, "bin", "go")
    59  }
    60  
    61  // Reset puts the scan back at the beginning.
    62  func (d *Dirs) Reset() {
    63  	d.offset = 0
    64  }
    65  
    66  // Next returns the next directory in the scan. The boolean
    67  // is false when the scan is done.
    68  func (d *Dirs) Next() (Dir, bool) {
    69  	if d.offset < len(d.hist) {
    70  		dir := d.hist[d.offset]
    71  		d.offset++
    72  		return dir, true
    73  	}
    74  	dir, ok := <-d.scan
    75  	if !ok {
    76  		return Dir{}, false
    77  	}
    78  	d.hist = append(d.hist, dir)
    79  	d.offset++
    80  	return dir, ok
    81  }
    82  
    83  // walk walks the trees in GOROOT and GOPATH.
    84  func (d *Dirs) walk(roots []Dir) {
    85  	for _, root := range roots {
    86  		d.bfsWalkRoot(root)
    87  	}
    88  	close(d.scan)
    89  }
    90  
    91  // bfsWalkRoot walks a single directory hierarchy in breadth-first lexical order.
    92  // Each Go source directory it finds is delivered on d.scan.
    93  func (d *Dirs) bfsWalkRoot(root Dir) {
    94  	root.dir = filepath.Clean(root.dir) // because filepath.Join will do it anyway
    95  
    96  	// this is the queue of directories to examine in this pass.
    97  	this := []string{}
    98  	// next is the queue of directories to examine in the next pass.
    99  	next := []string{root.dir}
   100  
   101  	for len(next) > 0 {
   102  		this, next = next, this[0:0]
   103  		for _, dir := range this {
   104  			fd, err := os.Open(dir)
   105  			if err != nil {
   106  				log.Print(err)
   107  				continue
   108  			}
   109  			entries, err := fd.Readdir(0)
   110  			fd.Close()
   111  			if err != nil {
   112  				log.Print(err)
   113  				continue
   114  			}
   115  			hasGoFiles := false
   116  			for _, entry := range entries {
   117  				name := entry.Name()
   118  				// For plain files, remember if this directory contains any .go
   119  				// source files, but ignore them otherwise.
   120  				if !entry.IsDir() {
   121  					if !hasGoFiles && strings.HasSuffix(name, ".go") {
   122  						hasGoFiles = true
   123  					}
   124  					continue
   125  				}
   126  				// Entry is a directory.
   127  
   128  				// The go tool ignores directories starting with ., _, or named "testdata".
   129  				if name[0] == '.' || name[0] == '_' || name == "testdata" {
   130  					continue
   131  				}
   132  				// When in a module, ignore vendor directories and stop at module boundaries.
   133  				if root.inModule {
   134  					if name == "vendor" {
   135  						continue
   136  					}
   137  					if fi, err := os.Stat(filepath.Join(dir, name, "go.mod")); err == nil && !fi.IsDir() {
   138  						continue
   139  					}
   140  				}
   141  				// Remember this (fully qualified) directory for the next pass.
   142  				next = append(next, filepath.Join(dir, name))
   143  			}
   144  			if hasGoFiles {
   145  				// It's a candidate.
   146  				importPath := root.importPath
   147  				if len(dir) > len(root.dir) {
   148  					if importPath != "" {
   149  						importPath += "/"
   150  					}
   151  					importPath += filepath.ToSlash(dir[len(root.dir)+1:])
   152  				}
   153  				d.scan <- Dir{importPath, dir, root.inModule}
   154  			}
   155  		}
   156  
   157  	}
   158  }
   159  
   160  var testGOPATH = false // force GOPATH use for testing
   161  
   162  // codeRoots returns the code roots to search for packages.
   163  // In GOPATH mode this is GOROOT/src and GOPATH/src, with empty import paths.
   164  // In module mode, this is each module root, with an import path set to its module path.
   165  func codeRoots() []Dir {
   166  	codeRootsCache.once.Do(func() {
   167  		codeRootsCache.roots = findCodeRoots()
   168  	})
   169  	return codeRootsCache.roots
   170  }
   171  
   172  var codeRootsCache struct {
   173  	once  sync.Once
   174  	roots []Dir
   175  }
   176  
   177  var usingModules bool
   178  
   179  func findCodeRoots() []Dir {
   180  	var list []Dir
   181  	if !testGOPATH {
   182  		// TODO: use the same state used to load the package.
   183  		// For now it's okay to use a new state because we're just
   184  		// using it to determine whether we're in module mode. But
   185  		// it would be good to avoid an extra run of modload.Init.
   186  		if state := modload.NewLoader(); state.WillBeEnabled() {
   187  			usingModules = state.HasModRoot()
   188  
   189  			if cfg.GOROOT != "" {
   190  				list = append(list,
   191  					Dir{dir: filepath.Join(cfg.GOROOT, "src"), inModule: usingModules},
   192  					Dir{importPath: "cmd", dir: filepath.Join(cfg.GOROOT, "src", "cmd"), inModule: usingModules})
   193  			}
   194  
   195  			if !usingModules {
   196  				// Modules are enabled, but the working directory is outside any module.
   197  				// We can still access std, cmd, and packages specified as source files
   198  				// on the command line, but there are no module roots.
   199  				// Avoid 'go list -m all' below, since it will not work.
   200  				return list
   201  			}
   202  		}
   203  	}
   204  
   205  	if !usingModules {
   206  		if cfg.GOROOT != "" {
   207  			list = append(list, Dir{dir: filepath.Join(cfg.GOROOT, "src")})
   208  		}
   209  		for _, root := range splitGopath() {
   210  			list = append(list, Dir{dir: filepath.Join(root, "src")})
   211  		}
   212  		return list
   213  	}
   214  
   215  	// Find module root directories from go list.
   216  	// Eventually we want golang.org/x/tools/go/packages
   217  	// to handle the entire file system search and become go/packages,
   218  	// but for now enumerating the module roots lets us fit modules
   219  	// into the current code with as few changes as possible.
   220  	mainMod, vendorEnabled, err := vendorEnabled()
   221  	if err != nil {
   222  		return list
   223  	}
   224  	if vendorEnabled {
   225  		// Add the vendor directory to the search path ahead of "std".
   226  		// That way, if the main module *is* "std", we will identify the path
   227  		// without the "vendor/" prefix before the one with that prefix.
   228  		list = append([]Dir{{dir: filepath.Join(mainMod.Dir, "vendor"), inModule: false}}, list...)
   229  		if mainMod.Path != "std" {
   230  			list = append(list, Dir{importPath: mainMod.Path, dir: mainMod.Dir, inModule: true})
   231  		}
   232  		return list
   233  	}
   234  
   235  	cmd := exec.Command(goCmd(), "list", "-m", "-f={{.Path}}\t{{.Dir}}", "all")
   236  	cmd.Stderr = os.Stderr
   237  	out, _ := cmd.Output()
   238  	for line := range strings.SplitSeq(string(out), "\n") {
   239  		path, dir, _ := strings.Cut(line, "\t")
   240  		if dir != "" {
   241  			list = append(list, Dir{importPath: path, dir: dir, inModule: true})
   242  		}
   243  	}
   244  
   245  	return list
   246  }
   247  
   248  // The functions below are derived from x/tools/internal/imports at CL 203017.
   249  
   250  type moduleJSON struct {
   251  	Path, Dir, GoVersion string
   252  }
   253  
   254  var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
   255  
   256  // vendorEnabled indicates if vendoring is enabled.
   257  // Inspired by setDefaultBuildMod in modload/init.go
   258  func vendorEnabled() (*moduleJSON, bool, error) {
   259  	mainMod, go114, err := getMainModuleAnd114()
   260  	if err != nil {
   261  		return nil, false, err
   262  	}
   263  
   264  	stdout, _ := exec.Command(goCmd(), "env", "GOFLAGS").Output()
   265  	goflags := string(bytes.TrimSpace(stdout))
   266  	matches := modFlagRegexp.FindStringSubmatch(goflags)
   267  	var modFlag string
   268  	if len(matches) != 0 {
   269  		modFlag = matches[1]
   270  	}
   271  	if modFlag != "" {
   272  		// Don't override an explicit '-mod=' argument.
   273  		return mainMod, modFlag == "vendor", nil
   274  	}
   275  	if mainMod == nil || !go114 {
   276  		return mainMod, false, nil
   277  	}
   278  	// Check 1.14's automatic vendor mode.
   279  	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
   280  		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
   281  			// The Go version is at least 1.14, and a vendor directory exists.
   282  			// Set -mod=vendor by default.
   283  			return mainMod, true, nil
   284  		}
   285  	}
   286  	return mainMod, false, nil
   287  }
   288  
   289  // getMainModuleAnd114 gets the main module's information and whether the
   290  // go command in use is 1.14+. This is the information needed to figure out
   291  // if vendoring should be enabled.
   292  func getMainModuleAnd114() (*moduleJSON, bool, error) {
   293  	const format = `{{.Path}}
   294  {{.Dir}}
   295  {{.GoVersion}}
   296  {{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
   297  `
   298  	cmd := exec.Command(goCmd(), "list", "-m", "-f", format)
   299  	cmd.Stderr = os.Stderr
   300  	stdout, err := cmd.Output()
   301  	if err != nil {
   302  		return nil, false, nil
   303  	}
   304  	lines := strings.Split(string(stdout), "\n")
   305  	if len(lines) < 5 {
   306  		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
   307  	}
   308  	mod := &moduleJSON{
   309  		Path:      lines[0],
   310  		Dir:       lines[1],
   311  		GoVersion: lines[2],
   312  	}
   313  	return mod, lines[3] == "go1.14", nil
   314  }
   315
View as plain text