dirs.go

Documentation: cmd/doc

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"log"
    11  	"os"
    12  	"os/exec"
    13  	"path/filepath"
    14  	"regexp"
    15  	"strings"
    16  	"sync"
    17  
    18  	"golang.org/x/mod/semver"
    19  )
    20  
    21  // A Dir describes a directory holding code by specifying
    22  // the expected import path and the file system directory.
    23  type Dir struct {
    24  	importPath string // import path for that dir
    25  	dir        string // file system directory
    26  	inModule   bool
    27  }
    28  
    29  // Dirs is a structure for scanning the directory tree.
    30  // Its Next method returns the next Go source directory it finds.
    31  // Although it can be used to scan the tree multiple times, it
    32  // only walks the tree once, caching the data it finds.
    33  type Dirs struct {
    34  	scan   chan Dir // Directories generated by walk.
    35  	hist   []Dir    // History of reported Dirs.
    36  	offset int      // Counter for Next.
    37  }
    38  
    39  var dirs Dirs
    40  
    41  // dirsInit starts the scanning of package directories in GOROOT and GOPATH. Any
    42  // extra paths passed to it are included in the channel.
    43  func dirsInit(extra ...Dir) {
    44  	if buildCtx.GOROOT == "" {
    45  		stdout, err := exec.Command("go", "env", "GOROOT").Output()
    46  		if err != nil {
    47  			if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
    48  				log.Fatalf("failed to determine GOROOT: $GOROOT is not set and 'go env GOROOT' failed:\n%s", ee.Stderr)
    49  			}
    50  			log.Fatalf("failed to determine GOROOT: $GOROOT is not set and could not run 'go env GOROOT':\n\t%s", err)
    51  		}
    52  		buildCtx.GOROOT = string(bytes.TrimSpace(stdout))
    53  	}
    54  
    55  	dirs.hist = make([]Dir, 0, 1000)
    56  	dirs.hist = append(dirs.hist, extra...)
    57  	dirs.scan = make(chan Dir)
    58  	go dirs.walk(codeRoots())
    59  }
    60  
    61  // goCmd returns the "go" command path corresponding to buildCtx.GOROOT.
    62  func goCmd() string {
    63  	if buildCtx.GOROOT == "" {
    64  		return "go"
    65  	}
    66  	return filepath.Join(buildCtx.GOROOT, "bin", "go")
    67  }
    68  
    69  // Reset puts the scan back at the beginning.
    70  func (d *Dirs) Reset() {
    71  	d.offset = 0
    72  }
    73  
    74  // Next returns the next directory in the scan. The boolean
    75  // is false when the scan is done.
    76  func (d *Dirs) Next() (Dir, bool) {
    77  	if d.offset < len(d.hist) {
    78  		dir := d.hist[d.offset]
    79  		d.offset++
    80  		return dir, true
    81  	}
    82  	dir, ok := <-d.scan
    83  	if !ok {
    84  		return Dir{}, false
    85  	}
    86  	d.hist = append(d.hist, dir)
    87  	d.offset++
    88  	return dir, ok
    89  }
    90  
    91  // walk walks the trees in GOROOT and GOPATH.
    92  func (d *Dirs) walk(roots []Dir) {
    93  	for _, root := range roots {
    94  		d.bfsWalkRoot(root)
    95  	}
    96  	close(d.scan)
    97  }
    98  
    99  // bfsWalkRoot walks a single directory hierarchy in breadth-first lexical order.
   100  // Each Go source directory it finds is delivered on d.scan.
   101  func (d *Dirs) bfsWalkRoot(root Dir) {
   102  	root.dir = filepath.Clean(root.dir) // because filepath.Join will do it anyway
   103  
   104  	// this is the queue of directories to examine in this pass.
   105  	this := []string{}
   106  	// next is the queue of directories to examine in the next pass.
   107  	next := []string{root.dir}
   108  
   109  	for len(next) > 0 {
   110  		this, next = next, this[0:0]
   111  		for _, dir := range this {
   112  			fd, err := os.Open(dir)
   113  			if err != nil {
   114  				log.Print(err)
   115  				continue
   116  			}
   117  			entries, err := fd.Readdir(0)
   118  			fd.Close()
   119  			if err != nil {
   120  				log.Print(err)
   121  				continue
   122  			}
   123  			hasGoFiles := false
   124  			for _, entry := range entries {
   125  				name := entry.Name()
   126  				// For plain files, remember if this directory contains any .go
   127  				// source files, but ignore them otherwise.
   128  				if !entry.IsDir() {
   129  					if !hasGoFiles && strings.HasSuffix(name, ".go") {
   130  						hasGoFiles = true
   131  					}
   132  					continue
   133  				}
   134  				// Entry is a directory.
   135  
   136  				// The go tool ignores directories starting with ., _, or named "testdata".
   137  				if name[0] == '.' || name[0] == '_' || name == "testdata" {
   138  					continue
   139  				}
   140  				// When in a module, ignore vendor directories and stop at module boundaries.
   141  				if root.inModule {
   142  					if name == "vendor" {
   143  						continue
   144  					}
   145  					if fi, err := os.Stat(filepath.Join(dir, name, "go.mod")); err == nil && !fi.IsDir() {
   146  						continue
   147  					}
   148  				}
   149  				// Remember this (fully qualified) directory for the next pass.
   150  				next = append(next, filepath.Join(dir, name))
   151  			}
   152  			if hasGoFiles {
   153  				// It's a candidate.
   154  				importPath := root.importPath
   155  				if len(dir) > len(root.dir) {
   156  					if importPath != "" {
   157  						importPath += "/"
   158  					}
   159  					importPath += filepath.ToSlash(dir[len(root.dir)+1:])
   160  				}
   161  				d.scan <- Dir{importPath, dir, root.inModule}
   162  			}
   163  		}
   164  
   165  	}
   166  }
   167  
   168  var testGOPATH = false // force GOPATH use for testing
   169  
   170  // codeRoots returns the code roots to search for packages.
   171  // In GOPATH mode this is GOROOT/src and GOPATH/src, with empty import paths.
   172  // In module mode, this is each module root, with an import path set to its module path.
   173  func codeRoots() []Dir {
   174  	codeRootsCache.once.Do(func() {
   175  		codeRootsCache.roots = findCodeRoots()
   176  	})
   177  	return codeRootsCache.roots
   178  }
   179  
   180  var codeRootsCache struct {
   181  	once  sync.Once
   182  	roots []Dir
   183  }
   184  
   185  var usingModules bool
   186  
   187  func findCodeRoots() []Dir {
   188  	var list []Dir
   189  	if !testGOPATH {
   190  		// Check for use of modules by 'go env GOMOD',
   191  		// which reports a go.mod file path if modules are enabled.
   192  		stdout, _ := exec.Command(goCmd(), "env", "GOMOD").Output()
   193  		gomod := string(bytes.TrimSpace(stdout))
   194  
   195  		usingModules = len(gomod) > 0
   196  		if usingModules && buildCtx.GOROOT != "" {
   197  			list = append(list,
   198  				Dir{dir: filepath.Join(buildCtx.GOROOT, "src"), inModule: true},
   199  				Dir{importPath: "cmd", dir: filepath.Join(buildCtx.GOROOT, "src", "cmd"), inModule: true})
   200  		}
   201  
   202  		if gomod == os.DevNull {
   203  			// Modules are enabled, but the working directory is outside any module.
   204  			// We can still access std, cmd, and packages specified as source files
   205  			// on the command line, but there are no module roots.
   206  			// Avoid 'go list -m all' below, since it will not work.
   207  			return list
   208  		}
   209  	}
   210  
   211  	if !usingModules {
   212  		if buildCtx.GOROOT != "" {
   213  			list = append(list, Dir{dir: filepath.Join(buildCtx.GOROOT, "src")})
   214  		}
   215  		for _, root := range splitGopath() {
   216  			list = append(list, Dir{dir: filepath.Join(root, "src")})
   217  		}
   218  		return list
   219  	}
   220  
   221  	// Find module root directories from go list.
   222  	// Eventually we want golang.org/x/tools/go/packages
   223  	// to handle the entire file system search and become go/packages,
   224  	// but for now enumerating the module roots lets us fit modules
   225  	// into the current code with as few changes as possible.
   226  	mainMod, vendorEnabled, err := vendorEnabled()
   227  	if err != nil {
   228  		return list
   229  	}
   230  	if vendorEnabled {
   231  		// Add the vendor directory to the search path ahead of "std".
   232  		// That way, if the main module *is* "std", we will identify the path
   233  		// without the "vendor/" prefix before the one with that prefix.
   234  		list = append([]Dir{{dir: filepath.Join(mainMod.Dir, "vendor"), inModule: false}}, list...)
   235  		if mainMod.Path != "std" {
   236  			list = append(list, Dir{importPath: mainMod.Path, dir: mainMod.Dir, inModule: true})
   237  		}
   238  		return list
   239  	}
   240  
   241  	cmd := exec.Command(goCmd(), "list", "-m", "-f={{.Path}}\t{{.Dir}}", "all")
   242  	cmd.Stderr = os.Stderr
   243  	out, _ := cmd.Output()
   244  	for _, line := range strings.Split(string(out), "\n") {
   245  		path, dir, _ := strings.Cut(line, "\t")
   246  		if dir != "" {
   247  			list = append(list, Dir{importPath: path, dir: dir, inModule: true})
   248  		}
   249  	}
   250  
   251  	return list
   252  }
   253  
   254  // The functions below are derived from x/tools/internal/imports at CL 203017.
   255  
   256  type moduleJSON struct {
   257  	Path, Dir, GoVersion string
   258  }
   259  
   260  var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
   261  
   262  // vendorEnabled indicates if vendoring is enabled.
   263  // Inspired by setDefaultBuildMod in modload/init.go
   264  func vendorEnabled() (*moduleJSON, bool, error) {
   265  	mainMod, go114, err := getMainModuleAnd114()
   266  	if err != nil {
   267  		return nil, false, err
   268  	}
   269  
   270  	stdout, _ := exec.Command(goCmd(), "env", "GOFLAGS").Output()
   271  	goflags := string(bytes.TrimSpace(stdout))
   272  	matches := modFlagRegexp.FindStringSubmatch(goflags)
   273  	var modFlag string
   274  	if len(matches) != 0 {
   275  		modFlag = matches[1]
   276  	}
   277  	if modFlag != "" {
   278  		// Don't override an explicit '-mod=' argument.
   279  		return mainMod, modFlag == "vendor", nil
   280  	}
   281  	if mainMod == nil || !go114 {
   282  		return mainMod, false, nil
   283  	}
   284  	// Check 1.14's automatic vendor mode.
   285  	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
   286  		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
   287  			// The Go version is at least 1.14, and a vendor directory exists.
   288  			// Set -mod=vendor by default.
   289  			return mainMod, true, nil
   290  		}
   291  	}
   292  	return mainMod, false, nil
   293  }
   294  
   295  // getMainModuleAnd114 gets the main module's information and whether the
   296  // go command in use is 1.14+. This is the information needed to figure out
   297  // if vendoring should be enabled.
   298  func getMainModuleAnd114() (*moduleJSON, bool, error) {
   299  	const format = `{{.Path}}
   300  {{.Dir}}
   301  {{.GoVersion}}
   302  {{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
   303  `
   304  	cmd := exec.Command(goCmd(), "list", "-m", "-f", format)
   305  	cmd.Stderr = os.Stderr
   306  	stdout, err := cmd.Output()
   307  	if err != nil {
   308  		return nil, false, nil
   309  	}
   310  	lines := strings.Split(string(stdout), "\n")
   311  	if len(lines) < 5 {
   312  		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
   313  	}
   314  	mod := &moduleJSON{
   315  		Path:      lines[0],
   316  		Dir:       lines[1],
   317  		GoVersion: lines[2],
   318  	}
   319  	return mod, lines[3] == "go1.14", nil
   320  }
   321
View as plain text