Source file src/cmd/go/internal/modindex/read.go

Documentation: cmd/go/internal/modindex

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modindex
     6  
     7  import (
     8  	"bytes"
     9  	"encoding/binary"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"go/build/constraint"
    14  	"go/token"
    15  	"internal/godebug"
    16  	"internal/goroot"
    17  	"path"
    18  	"path/filepath"
    19  	"runtime"
    20  	"runtime/debug"
    21  	"sort"
    22  	"strings"
    23  	"sync"
    24  	"time"
    25  	"unsafe"
    26  
    27  	"cmd/go/internal/base"
    28  	"cmd/go/internal/cache"
    29  	"cmd/go/internal/cfg"
    30  	"cmd/go/internal/fsys"
    31  	"cmd/go/internal/imports"
    32  	"cmd/go/internal/par"
    33  	"cmd/go/internal/str"
    34  )
    35  
    36  // enabled is used to flag off the behavior of the module index on tip.
    37  // It will be removed before the release.
    38  // TODO(matloob): Remove enabled once we have more confidence on the
    39  // module index.
    40  var enabled = godebug.New("#goindex").Value() != "0"
    41  
    42  // Module represents and encoded module index file. It is used to
    43  // do the equivalent of build.Import of packages in the module and answer other
    44  // questions based on the index file's data.
    45  type Module struct {
    46  	modroot string
    47  	d       *decoder
    48  	n       int // number of packages
    49  }
    50  
    51  // moduleHash returns an ActionID corresponding to the state of the module
    52  // located at filesystem path modroot.
    53  func moduleHash(modroot string, ismodcache bool) (cache.ActionID, error) {
    54  	// We expect modules stored within the module cache to be checksummed and
    55  	// immutable, and we expect released modules within GOROOT to change only
    56  	// infrequently (when the Go version changes).
    57  	if !ismodcache {
    58  		// The contents of this module may change over time. We don't want to pay
    59  		// the cost to detect changes and re-index whenever they occur, so just
    60  		// don't index it at all.
    61  		//
    62  		// Note that this is true even for modules in GOROOT/src: non-release builds
    63  		// of the Go toolchain may have arbitrary development changes on top of the
    64  		// commit reported by runtime.Version, or could be completely artificial due
    65  		// to lacking a `git` binary (like "devel gomote.XXXXX", as synthesized by
    66  		// "gomote push" as of 2022-06-15). (Release builds shouldn't have
    67  		// modifications, but we don't want to use a behavior for releases that we
    68  		// haven't tested during development.)
    69  		return cache.ActionID{}, ErrNotIndexed
    70  	}
    71  
    72  	h := cache.NewHash("moduleIndex")
    73  	// TODO(bcmills): Since modules in the index are checksummed, we could
    74  	// probably improve the cache hit rate by keying off of the module
    75  	// path@version (perhaps including the checksum?) instead of the module root
    76  	// directory.
    77  	fmt.Fprintf(h, "module index %s %s %v\n", runtime.Version(), indexVersion, modroot)
    78  	return h.Sum(), nil
    79  }
    80  
    81  const modTimeCutoff = 2 * time.Second
    82  
    83  // dirHash returns an ActionID corresponding to the state of the package
    84  // located at filesystem path pkgdir.
    85  func dirHash(modroot, pkgdir string) (cache.ActionID, error) {
    86  	h := cache.NewHash("moduleIndex")
    87  	fmt.Fprintf(h, "modroot %s\n", modroot)
    88  	fmt.Fprintf(h, "package %s %s %v\n", runtime.Version(), indexVersion, pkgdir)
    89  	entries, err := fsys.ReadDir(pkgdir)
    90  	if err != nil {
    91  		// pkgdir might not be a directory. give up on hashing.
    92  		return cache.ActionID{}, ErrNotIndexed
    93  	}
    94  	cutoff := time.Now().Add(-modTimeCutoff)
    95  	for _, info := range entries {
    96  		if info.IsDir() {
    97  			continue
    98  		}
    99  
   100  		if !info.Mode().IsRegular() {
   101  			return cache.ActionID{}, ErrNotIndexed
   102  		}
   103  		// To avoid problems for very recent files where a new
   104  		// write might not change the mtime due to file system
   105  		// mtime precision, reject caching if a file was read that
   106  		// is less than modTimeCutoff old.
   107  		//
   108  		// This is the same strategy used for hashing test inputs.
   109  		// See hashOpen in cmd/go/internal/test/test.go for the
   110  		// corresponding code.
   111  		if info.ModTime().After(cutoff) {
   112  			return cache.ActionID{}, ErrNotIndexed
   113  		}
   114  
   115  		fmt.Fprintf(h, "file %v %v %v\n", info.Name(), info.ModTime(), info.Size())
   116  	}
   117  	return h.Sum(), nil
   118  }
   119  
   120  var ErrNotIndexed = errors.New("not in module index")
   121  
   122  var (
   123  	errDisabled           = fmt.Errorf("%w: module indexing disabled", ErrNotIndexed)
   124  	errNotFromModuleCache = fmt.Errorf("%w: not from module cache", ErrNotIndexed)
   125  )
   126  
   127  // GetPackage returns the IndexPackage for the directory at the given path.
   128  // It will return ErrNotIndexed if the directory should be read without
   129  // using the index, for instance because the index is disabled, or the package
   130  // is not in a module.
   131  func GetPackage(modroot, pkgdir string) (*IndexPackage, error) {
   132  	mi, err := GetModule(modroot)
   133  	if err == nil {
   134  		return mi.Package(relPath(pkgdir, modroot)), nil
   135  	}
   136  	if !errors.Is(err, errNotFromModuleCache) {
   137  		return nil, err
   138  	}
   139  	if cfg.BuildContext.Compiler == "gccgo" && str.HasPathPrefix(modroot, cfg.GOROOTsrc) {
   140  		return nil, err // gccgo has no sources for GOROOT packages.
   141  	}
   142  	return openIndexPackage(modroot, pkgdir)
   143  }
   144  
   145  // GetModule returns the Module for the given modroot.
   146  // It will return ErrNotIndexed if the directory should be read without
   147  // using the index, for instance because the index is disabled, or the package
   148  // is not in a module.
   149  func GetModule(modroot string) (*Module, error) {
   150  	dir, _ := cache.DefaultDir()
   151  	if !enabled || dir == "off" {
   152  		return nil, errDisabled
   153  	}
   154  	if modroot == "" {
   155  		panic("modindex.GetPackage called with empty modroot")
   156  	}
   157  	if cfg.BuildMod == "vendor" {
   158  		// Even if the main module is in the module cache,
   159  		// its vendored dependencies are not loaded from their
   160  		// usual cached locations.
   161  		return nil, errNotFromModuleCache
   162  	}
   163  	modroot = filepath.Clean(modroot)
   164  	if str.HasFilePathPrefix(modroot, cfg.GOROOTsrc) || !str.HasFilePathPrefix(modroot, cfg.GOMODCACHE) {
   165  		return nil, errNotFromModuleCache
   166  	}
   167  	return openIndexModule(modroot, true)
   168  }
   169  
   170  var mcache par.ErrCache[string, *Module]
   171  
   172  // openIndexModule returns the module index for modPath.
   173  // It will return ErrNotIndexed if the module can not be read
   174  // using the index because it contains symlinks.
   175  func openIndexModule(modroot string, ismodcache bool) (*Module, error) {
   176  	return mcache.Do(modroot, func() (*Module, error) {
   177  		fsys.Trace("openIndexModule", modroot)
   178  		id, err := moduleHash(modroot, ismodcache)
   179  		if err != nil {
   180  			return nil, err
   181  		}
   182  		data, _, err := cache.GetMmap(cache.Default(), id)
   183  		if err != nil {
   184  			// Couldn't read from modindex. Assume we couldn't read from
   185  			// the index because the module hasn't been indexed yet.
   186  			data, err = indexModule(modroot)
   187  			if err != nil {
   188  				return nil, err
   189  			}
   190  			if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   191  				return nil, err
   192  			}
   193  		}
   194  		mi, err := fromBytes(modroot, data)
   195  		if err != nil {
   196  			return nil, err
   197  		}
   198  		return mi, nil
   199  	})
   200  }
   201  
   202  var pcache par.ErrCache[[2]string, *IndexPackage]
   203  
   204  func openIndexPackage(modroot, pkgdir string) (*IndexPackage, error) {
   205  	return pcache.Do([2]string{modroot, pkgdir}, func() (*IndexPackage, error) {
   206  		fsys.Trace("openIndexPackage", pkgdir)
   207  		id, err := dirHash(modroot, pkgdir)
   208  		if err != nil {
   209  			return nil, err
   210  		}
   211  		data, _, err := cache.GetMmap(cache.Default(), id)
   212  		if err != nil {
   213  			// Couldn't read from index. Assume we couldn't read from
   214  			// the index because the package hasn't been indexed yet.
   215  			data = indexPackage(modroot, pkgdir)
   216  			if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   217  				return nil, err
   218  			}
   219  		}
   220  		pkg, err := packageFromBytes(modroot, data)
   221  		if err != nil {
   222  			return nil, err
   223  		}
   224  		return pkg, nil
   225  	})
   226  }
   227  
   228  var errCorrupt = errors.New("corrupt index")
   229  
   230  // protect marks the start of a large section of code that accesses the index.
   231  // It should be used as:
   232  //
   233  //	defer unprotect(protect, &err)
   234  //
   235  // It should not be used for trivial accesses which would be
   236  // dwarfed by the overhead of the defer.
   237  func protect() bool {
   238  	return debug.SetPanicOnFault(true)
   239  }
   240  
   241  var isTest = false
   242  
   243  // unprotect marks the end of a large section of code that accesses the index.
   244  // It should be used as:
   245  //
   246  //	defer unprotect(protect, &err)
   247  //
   248  // end looks for panics due to errCorrupt or bad mmap accesses.
   249  // When it finds them, it adds explanatory text, consumes the panic, and sets *errp instead.
   250  // If errp is nil, end adds the explanatory text but then calls base.Fatalf.
   251  func unprotect(old bool, errp *error) {
   252  	// SetPanicOnFault's errors _may_ satisfy this interface. Even though it's not guaranteed
   253  	// that all its errors satisfy this interface, we'll only check for these errors so that
   254  	// we don't suppress panics that could have been produced from other sources.
   255  	type addrer interface {
   256  		Addr() uintptr
   257  	}
   258  
   259  	debug.SetPanicOnFault(old)
   260  
   261  	if e := recover(); e != nil {
   262  		if _, ok := e.(addrer); ok || e == errCorrupt {
   263  			// This panic was almost certainly caused by SetPanicOnFault or our panic(errCorrupt).
   264  			err := fmt.Errorf("error reading module index: %v", e)
   265  			if errp != nil {
   266  				*errp = err
   267  				return
   268  			}
   269  			if isTest {
   270  				panic(err)
   271  			}
   272  			base.Fatalf("%v", err)
   273  		}
   274  		// The panic was likely not caused by SetPanicOnFault.
   275  		panic(e)
   276  	}
   277  }
   278  
   279  // fromBytes returns a *Module given the encoded representation.
   280  func fromBytes(moddir string, data []byte) (m *Module, err error) {
   281  	if !enabled {
   282  		panic("use of index")
   283  	}
   284  
   285  	defer unprotect(protect(), &err)
   286  
   287  	if !bytes.HasPrefix(data, []byte(indexVersion+"\n")) {
   288  		return nil, errCorrupt
   289  	}
   290  
   291  	const hdr = len(indexVersion + "\n")
   292  	d := &decoder{data: data}
   293  	str := d.intAt(hdr)
   294  	if str < hdr+8 || len(d.data) < str {
   295  		return nil, errCorrupt
   296  	}
   297  	d.data, d.str = data[:str], d.data[str:]
   298  	// Check that string table looks valid.
   299  	// First string is empty string (length 0),
   300  	// and we leave a marker byte 0xFF at the end
   301  	// just to make sure that the file is not truncated.
   302  	if len(d.str) == 0 || d.str[0] != 0 || d.str[len(d.str)-1] != 0xFF {
   303  		return nil, errCorrupt
   304  	}
   305  
   306  	n := d.intAt(hdr + 4)
   307  	if n < 0 || n > (len(d.data)-8)/8 {
   308  		return nil, errCorrupt
   309  	}
   310  
   311  	m = &Module{
   312  		moddir,
   313  		d,
   314  		n,
   315  	}
   316  	return m, nil
   317  }
   318  
   319  // packageFromBytes returns a *IndexPackage given the encoded representation.
   320  func packageFromBytes(modroot string, data []byte) (p *IndexPackage, err error) {
   321  	m, err := fromBytes(modroot, data)
   322  	if err != nil {
   323  		return nil, err
   324  	}
   325  	if m.n != 1 {
   326  		return nil, fmt.Errorf("corrupt single-package index")
   327  	}
   328  	return m.pkg(0), nil
   329  }
   330  
   331  // pkgDir returns the dir string of the i'th package in the index.
   332  func (m *Module) pkgDir(i int) string {
   333  	if i < 0 || i >= m.n {
   334  		panic(errCorrupt)
   335  	}
   336  	return m.d.stringAt(12 + 8 + 8*i)
   337  }
   338  
   339  // pkgOff returns the offset of the data for the i'th package in the index.
   340  func (m *Module) pkgOff(i int) int {
   341  	if i < 0 || i >= m.n {
   342  		panic(errCorrupt)
   343  	}
   344  	return m.d.intAt(12 + 8 + 8*i + 4)
   345  }
   346  
   347  // Walk calls f for each package in the index, passing the path to that package relative to the module root.
   348  func (m *Module) Walk(f func(path string)) {
   349  	defer unprotect(protect(), nil)
   350  	for i := 0; i < m.n; i++ {
   351  		f(m.pkgDir(i))
   352  	}
   353  }
   354  
   355  // relPath returns the path relative to the module's root.
   356  func relPath(path, modroot string) string {
   357  	return str.TrimFilePathPrefix(filepath.Clean(path), filepath.Clean(modroot))
   358  }
   359  
   360  var installgorootAll = godebug.New("installgoroot").Value() == "all"
   361  
   362  // Import is the equivalent of build.Import given the information in Module.
   363  func (rp *IndexPackage) Import(bctxt build.Context, mode build.ImportMode) (p *build.Package, err error) {
   364  	defer unprotect(protect(), &err)
   365  
   366  	ctxt := (*Context)(&bctxt)
   367  
   368  	p = &build.Package{}
   369  
   370  	p.ImportPath = "."
   371  	p.Dir = filepath.Join(rp.modroot, rp.dir)
   372  
   373  	var pkgerr error
   374  	switch ctxt.Compiler {
   375  	case "gccgo", "gc":
   376  	default:
   377  		// Save error for end of function.
   378  		pkgerr = fmt.Errorf("import %q: unknown compiler %q", p.Dir, ctxt.Compiler)
   379  	}
   380  
   381  	if p.Dir == "" {
   382  		return p, fmt.Errorf("import %q: import of unknown directory", p.Dir)
   383  	}
   384  
   385  	// goroot and gopath
   386  	inTestdata := func(sub string) bool {
   387  		return strings.Contains(sub, "/testdata/") || strings.HasSuffix(sub, "/testdata") || str.HasPathPrefix(sub, "testdata")
   388  	}
   389  	var pkga string
   390  	if !inTestdata(rp.dir) {
   391  		// In build.go, p.Root should only be set in the non-local-import case, or in
   392  		// GOROOT or GOPATH. Since module mode only calls Import with path set to "."
   393  		// and the module index doesn't apply outside modules, the GOROOT case is
   394  		// the only case where p.Root needs to be set.
   395  		if ctxt.GOROOT != "" && str.HasFilePathPrefix(p.Dir, cfg.GOROOTsrc) && p.Dir != cfg.GOROOTsrc {
   396  			p.Root = ctxt.GOROOT
   397  			p.Goroot = true
   398  			modprefix := str.TrimFilePathPrefix(rp.modroot, cfg.GOROOTsrc)
   399  			p.ImportPath = rp.dir
   400  			if modprefix != "" {
   401  				p.ImportPath = filepath.Join(modprefix, p.ImportPath)
   402  			}
   403  
   404  			// Set GOROOT-specific fields (sometimes for modules in a GOPATH directory).
   405  			// The fields set below (SrcRoot, PkgRoot, BinDir, PkgTargetRoot, and PkgObj)
   406  			// are only set in build.Import if p.Root != "".
   407  			var pkgtargetroot string
   408  			suffix := ""
   409  			if ctxt.InstallSuffix != "" {
   410  				suffix = "_" + ctxt.InstallSuffix
   411  			}
   412  			switch ctxt.Compiler {
   413  			case "gccgo":
   414  				pkgtargetroot = "pkg/gccgo_" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   415  				dir, elem := path.Split(p.ImportPath)
   416  				pkga = pkgtargetroot + "/" + dir + "lib" + elem + ".a"
   417  			case "gc":
   418  				pkgtargetroot = "pkg/" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   419  				pkga = pkgtargetroot + "/" + p.ImportPath + ".a"
   420  			}
   421  			p.SrcRoot = ctxt.joinPath(p.Root, "src")
   422  			p.PkgRoot = ctxt.joinPath(p.Root, "pkg")
   423  			p.BinDir = ctxt.joinPath(p.Root, "bin")
   424  			if pkga != "" {
   425  				// Always set PkgTargetRoot. It might be used when building in shared
   426  				// mode.
   427  				p.PkgTargetRoot = ctxt.joinPath(p.Root, pkgtargetroot)
   428  
   429  				// Set the install target if applicable.
   430  				if !p.Goroot || (installgorootAll && p.ImportPath != "unsafe" && p.ImportPath != "builtin") {
   431  					p.PkgObj = ctxt.joinPath(p.Root, pkga)
   432  				}
   433  			}
   434  		}
   435  	}
   436  
   437  	if rp.error != nil {
   438  		if errors.Is(rp.error, errCannotFindPackage) && ctxt.Compiler == "gccgo" && p.Goroot {
   439  			return p, nil
   440  		}
   441  		return p, rp.error
   442  	}
   443  
   444  	if mode&build.FindOnly != 0 {
   445  		return p, pkgerr
   446  	}
   447  
   448  	// We need to do a second round of bad file processing.
   449  	var badGoError error
   450  	badGoFiles := make(map[string]bool)
   451  	badGoFile := func(name string, err error) {
   452  		if badGoError == nil {
   453  			badGoError = err
   454  		}
   455  		if !badGoFiles[name] {
   456  			p.InvalidGoFiles = append(p.InvalidGoFiles, name)
   457  			badGoFiles[name] = true
   458  		}
   459  	}
   460  
   461  	var Sfiles []string // files with ".S"(capital S)/.sx(capital s equivalent for case insensitive filesystems)
   462  	var firstFile string
   463  	embedPos := make(map[string][]token.Position)
   464  	testEmbedPos := make(map[string][]token.Position)
   465  	xTestEmbedPos := make(map[string][]token.Position)
   466  	importPos := make(map[string][]token.Position)
   467  	testImportPos := make(map[string][]token.Position)
   468  	xTestImportPos := make(map[string][]token.Position)
   469  	allTags := make(map[string]bool)
   470  	for _, tf := range rp.sourceFiles {
   471  		name := tf.name()
   472  		// Check errors for go files and call badGoFiles to put them in
   473  		// InvalidGoFiles if they do have an error.
   474  		if strings.HasSuffix(name, ".go") {
   475  			if error := tf.error(); error != "" {
   476  				badGoFile(name, errors.New(tf.error()))
   477  				continue
   478  			} else if parseError := tf.parseError(); parseError != "" {
   479  				badGoFile(name, parseErrorFromString(tf.parseError()))
   480  				// Fall through: we still want to list files with parse errors.
   481  			}
   482  		}
   483  
   484  		var shouldBuild = true
   485  		if !ctxt.goodOSArchFile(name, allTags) && !ctxt.UseAllFiles {
   486  			shouldBuild = false
   487  		} else if goBuildConstraint := tf.goBuildConstraint(); goBuildConstraint != "" {
   488  			x, err := constraint.Parse(goBuildConstraint)
   489  			if err != nil {
   490  				return p, fmt.Errorf("%s: parsing //go:build line: %v", name, err)
   491  			}
   492  			shouldBuild = ctxt.eval(x, allTags)
   493  		} else if plusBuildConstraints := tf.plusBuildConstraints(); len(plusBuildConstraints) > 0 {
   494  			for _, text := range plusBuildConstraints {
   495  				if x, err := constraint.Parse(text); err == nil {
   496  					if !ctxt.eval(x, allTags) {
   497  						shouldBuild = false
   498  					}
   499  				}
   500  			}
   501  		}
   502  
   503  		ext := nameExt(name)
   504  		if !shouldBuild || tf.ignoreFile() {
   505  			if ext == ".go" {
   506  				p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   507  			} else if fileListForExt(p, ext) != nil {
   508  				p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, name)
   509  			}
   510  			continue
   511  		}
   512  
   513  		// Going to save the file. For non-Go files, can stop here.
   514  		switch ext {
   515  		case ".go":
   516  			// keep going
   517  		case ".S", ".sx":
   518  			// special case for cgo, handled at end
   519  			Sfiles = append(Sfiles, name)
   520  			continue
   521  		default:
   522  			if list := fileListForExt(p, ext); list != nil {
   523  				*list = append(*list, name)
   524  			}
   525  			continue
   526  		}
   527  
   528  		pkg := tf.pkgName()
   529  		if pkg == "documentation" {
   530  			p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   531  			continue
   532  		}
   533  		isTest := strings.HasSuffix(name, "_test.go")
   534  		isXTest := false
   535  		if isTest && strings.HasSuffix(tf.pkgName(), "_test") && p.Name != tf.pkgName() {
   536  			isXTest = true
   537  			pkg = pkg[:len(pkg)-len("_test")]
   538  		}
   539  
   540  		if !isTest && tf.binaryOnly() {
   541  			p.BinaryOnly = true
   542  		}
   543  
   544  		if p.Name == "" {
   545  			p.Name = pkg
   546  			firstFile = name
   547  		} else if pkg != p.Name {
   548  			// TODO(#45999): The choice of p.Name is arbitrary based on file iteration
   549  			// order. Instead of resolving p.Name arbitrarily, we should clear out the
   550  			// existing Name and mark the existing files as also invalid.
   551  			badGoFile(name, &MultiplePackageError{
   552  				Dir:      p.Dir,
   553  				Packages: []string{p.Name, pkg},
   554  				Files:    []string{firstFile, name},
   555  			})
   556  		}
   557  		// Grab the first package comment as docs, provided it is not from a test file.
   558  		if p.Doc == "" && !isTest && !isXTest {
   559  			if synopsis := tf.synopsis(); synopsis != "" {
   560  				p.Doc = synopsis
   561  			}
   562  		}
   563  
   564  		// Record Imports and information about cgo.
   565  		isCgo := false
   566  		imports := tf.imports()
   567  		for _, imp := range imports {
   568  			if imp.path == "C" {
   569  				if isTest {
   570  					badGoFile(name, fmt.Errorf("use of cgo in test %s not supported", name))
   571  					continue
   572  				}
   573  				isCgo = true
   574  			}
   575  		}
   576  		if directives := tf.cgoDirectives(); directives != "" {
   577  			if err := ctxt.saveCgo(name, p, directives); err != nil {
   578  				badGoFile(name, err)
   579  			}
   580  		}
   581  
   582  		var fileList *[]string
   583  		var importMap, embedMap map[string][]token.Position
   584  		var directives *[]build.Directive
   585  		switch {
   586  		case isCgo:
   587  			allTags["cgo"] = true
   588  			if ctxt.CgoEnabled {
   589  				fileList = &p.CgoFiles
   590  				importMap = importPos
   591  				embedMap = embedPos
   592  				directives = &p.Directives
   593  			} else {
   594  				// Ignore Imports and Embeds from cgo files if cgo is disabled.
   595  				fileList = &p.IgnoredGoFiles
   596  			}
   597  		case isXTest:
   598  			fileList = &p.XTestGoFiles
   599  			importMap = xTestImportPos
   600  			embedMap = xTestEmbedPos
   601  			directives = &p.XTestDirectives
   602  		case isTest:
   603  			fileList = &p.TestGoFiles
   604  			importMap = testImportPos
   605  			embedMap = testEmbedPos
   606  			directives = &p.TestDirectives
   607  		default:
   608  			fileList = &p.GoFiles
   609  			importMap = importPos
   610  			embedMap = embedPos
   611  			directives = &p.Directives
   612  		}
   613  		*fileList = append(*fileList, name)
   614  		if importMap != nil {
   615  			for _, imp := range imports {
   616  				importMap[imp.path] = append(importMap[imp.path], imp.position)
   617  			}
   618  		}
   619  		if embedMap != nil {
   620  			for _, e := range tf.embeds() {
   621  				embedMap[e.pattern] = append(embedMap[e.pattern], e.position)
   622  			}
   623  		}
   624  		if directives != nil {
   625  			*directives = append(*directives, tf.directives()...)
   626  		}
   627  	}
   628  
   629  	p.EmbedPatterns, p.EmbedPatternPos = cleanDecls(embedPos)
   630  	p.TestEmbedPatterns, p.TestEmbedPatternPos = cleanDecls(testEmbedPos)
   631  	p.XTestEmbedPatterns, p.XTestEmbedPatternPos = cleanDecls(xTestEmbedPos)
   632  
   633  	p.Imports, p.ImportPos = cleanDecls(importPos)
   634  	p.TestImports, p.TestImportPos = cleanDecls(testImportPos)
   635  	p.XTestImports, p.XTestImportPos = cleanDecls(xTestImportPos)
   636  
   637  	for tag := range allTags {
   638  		p.AllTags = append(p.AllTags, tag)
   639  	}
   640  	sort.Strings(p.AllTags)
   641  
   642  	if len(p.CgoFiles) > 0 {
   643  		p.SFiles = append(p.SFiles, Sfiles...)
   644  		sort.Strings(p.SFiles)
   645  	} else {
   646  		p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, Sfiles...)
   647  		sort.Strings(p.IgnoredOtherFiles)
   648  	}
   649  
   650  	if badGoError != nil {
   651  		return p, badGoError
   652  	}
   653  	if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 {
   654  		return p, &build.NoGoError{Dir: p.Dir}
   655  	}
   656  	return p, pkgerr
   657  }
   658  
   659  // IsStandardPackage reports whether path is a standard package
   660  // for the goroot and compiler using the module index if possible,
   661  // and otherwise falling back to internal/goroot.IsStandardPackage
   662  func IsStandardPackage(goroot_, compiler, path string) bool {
   663  	if !enabled || compiler != "gc" {
   664  		return goroot.IsStandardPackage(goroot_, compiler, path)
   665  	}
   666  
   667  	reldir := filepath.FromSlash(path) // relative dir path in module index for package
   668  	modroot := filepath.Join(goroot_, "src")
   669  	if str.HasFilePathPrefix(reldir, "cmd") {
   670  		reldir = str.TrimFilePathPrefix(reldir, "cmd")
   671  		modroot = filepath.Join(modroot, "cmd")
   672  	}
   673  	if pkg, err := GetPackage(modroot, filepath.Join(modroot, reldir)); err == nil {
   674  		hasGo, err := pkg.IsDirWithGoFiles()
   675  		return err == nil && hasGo
   676  	} else if errors.Is(err, ErrNotIndexed) {
   677  		// Fall back because package isn't indexable. (Probably because
   678  		// a file was modified recently)
   679  		return goroot.IsStandardPackage(goroot_, compiler, path)
   680  	}
   681  	return false
   682  }
   683  
   684  // IsDirWithGoFiles is the equivalent of fsys.IsDirWithGoFiles using the information in the index.
   685  func (rp *IndexPackage) IsDirWithGoFiles() (_ bool, err error) {
   686  	defer func() {
   687  		if e := recover(); e != nil {
   688  			err = fmt.Errorf("error reading module index: %v", e)
   689  		}
   690  	}()
   691  	for _, sf := range rp.sourceFiles {
   692  		if strings.HasSuffix(sf.name(), ".go") {
   693  			return true, nil
   694  		}
   695  	}
   696  	return false, nil
   697  }
   698  
   699  // ScanDir implements imports.ScanDir using the information in the index.
   700  func (rp *IndexPackage) ScanDir(tags map[string]bool) (sortedImports []string, sortedTestImports []string, err error) {
   701  	// TODO(matloob) dir should eventually be relative to indexed directory
   702  	// TODO(matloob): skip reading raw package and jump straight to data we need?
   703  
   704  	defer func() {
   705  		if e := recover(); e != nil {
   706  			err = fmt.Errorf("error reading module index: %v", e)
   707  		}
   708  	}()
   709  
   710  	imports_ := make(map[string]bool)
   711  	testImports := make(map[string]bool)
   712  	numFiles := 0
   713  
   714  Files:
   715  	for _, sf := range rp.sourceFiles {
   716  		name := sf.name()
   717  		if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") || !strings.HasSuffix(name, ".go") || !imports.MatchFile(name, tags) {
   718  			continue
   719  		}
   720  
   721  		// The following section exists for backwards compatibility reasons:
   722  		// scanDir ignores files with import "C" when collecting the list
   723  		// of imports unless the "cgo" tag is provided. The following comment
   724  		// is copied from the original.
   725  		//
   726  		// import "C" is implicit requirement of cgo tag.
   727  		// When listing files on the command line (explicitFiles=true)
   728  		// we do not apply build tag filtering but we still do apply
   729  		// cgo filtering, so no explicitFiles check here.
   730  		// Why? Because we always have, and it's not worth breaking
   731  		// that behavior now.
   732  		imps := sf.imports() // TODO(matloob): directly read import paths to avoid the extra strings?
   733  		for _, imp := range imps {
   734  			if imp.path == "C" && !tags["cgo"] && !tags["*"] {
   735  				continue Files
   736  			}
   737  		}
   738  
   739  		if !shouldBuild(sf, tags) {
   740  			continue
   741  		}
   742  		numFiles++
   743  		m := imports_
   744  		if strings.HasSuffix(name, "_test.go") {
   745  			m = testImports
   746  		}
   747  		for _, p := range imps {
   748  			m[p.path] = true
   749  		}
   750  	}
   751  	if numFiles == 0 {
   752  		return nil, nil, imports.ErrNoGo
   753  	}
   754  	return keys(imports_), keys(testImports), nil
   755  }
   756  
   757  func keys(m map[string]bool) []string {
   758  	list := make([]string, 0, len(m))
   759  	for k := range m {
   760  		list = append(list, k)
   761  	}
   762  	sort.Strings(list)
   763  	return list
   764  }
   765  
   766  // implements imports.ShouldBuild in terms of an index sourcefile.
   767  func shouldBuild(sf *sourceFile, tags map[string]bool) bool {
   768  	if goBuildConstraint := sf.goBuildConstraint(); goBuildConstraint != "" {
   769  		x, err := constraint.Parse(goBuildConstraint)
   770  		if err != nil {
   771  			return false
   772  		}
   773  		return imports.Eval(x, tags, true)
   774  	}
   775  
   776  	plusBuildConstraints := sf.plusBuildConstraints()
   777  	for _, text := range plusBuildConstraints {
   778  		if x, err := constraint.Parse(text); err == nil {
   779  			if !imports.Eval(x, tags, true) {
   780  				return false
   781  			}
   782  		}
   783  	}
   784  
   785  	return true
   786  }
   787  
   788  // IndexPackage holds the information in the index
   789  // needed to load a package in a specific directory.
   790  type IndexPackage struct {
   791  	error error
   792  	dir   string // directory of the package relative to the modroot
   793  
   794  	modroot string
   795  
   796  	// Source files
   797  	sourceFiles []*sourceFile
   798  }
   799  
   800  var errCannotFindPackage = errors.New("cannot find package")
   801  
   802  // Package and returns finds the package with the given path (relative to the module root).
   803  // If the package does not exist, Package returns an IndexPackage that will return an
   804  // appropriate error from its methods.
   805  func (m *Module) Package(path string) *IndexPackage {
   806  	defer unprotect(protect(), nil)
   807  
   808  	i, ok := sort.Find(m.n, func(i int) int {
   809  		return strings.Compare(path, m.pkgDir(i))
   810  	})
   811  	if !ok {
   812  		return &IndexPackage{error: fmt.Errorf("%w %q in:\n\t%s", errCannotFindPackage, path, filepath.Join(m.modroot, path))}
   813  	}
   814  	return m.pkg(i)
   815  }
   816  
   817  // pkg returns the i'th IndexPackage in m.
   818  func (m *Module) pkg(i int) *IndexPackage {
   819  	r := m.d.readAt(m.pkgOff(i))
   820  	p := new(IndexPackage)
   821  	if errstr := r.string(); errstr != "" {
   822  		p.error = errors.New(errstr)
   823  	}
   824  	p.dir = r.string()
   825  	p.sourceFiles = make([]*sourceFile, r.int())
   826  	for i := range p.sourceFiles {
   827  		p.sourceFiles[i] = &sourceFile{
   828  			d:   m.d,
   829  			pos: r.int(),
   830  		}
   831  	}
   832  	p.modroot = m.modroot
   833  	return p
   834  }
   835  
   836  // sourceFile represents the information of a given source file in the module index.
   837  type sourceFile struct {
   838  	d               *decoder // encoding of this source file
   839  	pos             int      // start of sourceFile encoding in d
   840  	onceReadImports sync.Once
   841  	savedImports    []rawImport // saved imports so that they're only read once
   842  }
   843  
   844  // Offsets for fields in the sourceFile.
   845  const (
   846  	sourceFileError = 4 * iota
   847  	sourceFileParseError
   848  	sourceFileSynopsis
   849  	sourceFileName
   850  	sourceFilePkgName
   851  	sourceFileIgnoreFile
   852  	sourceFileBinaryOnly
   853  	sourceFileCgoDirectives
   854  	sourceFileGoBuildConstraint
   855  	sourceFileNumPlusBuildConstraints
   856  )
   857  
   858  func (sf *sourceFile) error() string {
   859  	return sf.d.stringAt(sf.pos + sourceFileError)
   860  }
   861  func (sf *sourceFile) parseError() string {
   862  	return sf.d.stringAt(sf.pos + sourceFileParseError)
   863  }
   864  func (sf *sourceFile) synopsis() string {
   865  	return sf.d.stringAt(sf.pos + sourceFileSynopsis)
   866  }
   867  func (sf *sourceFile) name() string {
   868  	return sf.d.stringAt(sf.pos + sourceFileName)
   869  }
   870  func (sf *sourceFile) pkgName() string {
   871  	return sf.d.stringAt(sf.pos + sourceFilePkgName)
   872  }
   873  func (sf *sourceFile) ignoreFile() bool {
   874  	return sf.d.boolAt(sf.pos + sourceFileIgnoreFile)
   875  }
   876  func (sf *sourceFile) binaryOnly() bool {
   877  	return sf.d.boolAt(sf.pos + sourceFileBinaryOnly)
   878  }
   879  func (sf *sourceFile) cgoDirectives() string {
   880  	return sf.d.stringAt(sf.pos + sourceFileCgoDirectives)
   881  }
   882  func (sf *sourceFile) goBuildConstraint() string {
   883  	return sf.d.stringAt(sf.pos + sourceFileGoBuildConstraint)
   884  }
   885  
   886  func (sf *sourceFile) plusBuildConstraints() []string {
   887  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   888  	n := sf.d.intAt(pos)
   889  	pos += 4
   890  	ret := make([]string, n)
   891  	for i := 0; i < n; i++ {
   892  		ret[i] = sf.d.stringAt(pos)
   893  		pos += 4
   894  	}
   895  	return ret
   896  }
   897  
   898  func (sf *sourceFile) importsOffset() int {
   899  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   900  	n := sf.d.intAt(pos)
   901  	// each build constraint is 1 uint32
   902  	return pos + 4 + n*4
   903  }
   904  
   905  func (sf *sourceFile) embedsOffset() int {
   906  	pos := sf.importsOffset()
   907  	n := sf.d.intAt(pos)
   908  	// each import is 5 uint32s (string + tokpos)
   909  	return pos + 4 + n*(4*5)
   910  }
   911  
   912  func (sf *sourceFile) directivesOffset() int {
   913  	pos := sf.embedsOffset()
   914  	n := sf.d.intAt(pos)
   915  	// each embed is 5 uint32s (string + tokpos)
   916  	return pos + 4 + n*(4*5)
   917  }
   918  
   919  func (sf *sourceFile) imports() []rawImport {
   920  	sf.onceReadImports.Do(func() {
   921  		importsOffset := sf.importsOffset()
   922  		r := sf.d.readAt(importsOffset)
   923  		numImports := r.int()
   924  		ret := make([]rawImport, numImports)
   925  		for i := 0; i < numImports; i++ {
   926  			ret[i] = rawImport{r.string(), r.tokpos()}
   927  		}
   928  		sf.savedImports = ret
   929  	})
   930  	return sf.savedImports
   931  }
   932  
   933  func (sf *sourceFile) embeds() []embed {
   934  	embedsOffset := sf.embedsOffset()
   935  	r := sf.d.readAt(embedsOffset)
   936  	numEmbeds := r.int()
   937  	ret := make([]embed, numEmbeds)
   938  	for i := range ret {
   939  		ret[i] = embed{r.string(), r.tokpos()}
   940  	}
   941  	return ret
   942  }
   943  
   944  func (sf *sourceFile) directives() []build.Directive {
   945  	directivesOffset := sf.directivesOffset()
   946  	r := sf.d.readAt(directivesOffset)
   947  	numDirectives := r.int()
   948  	ret := make([]build.Directive, numDirectives)
   949  	for i := range ret {
   950  		ret[i] = build.Directive{Text: r.string(), Pos: r.tokpos()}
   951  	}
   952  	return ret
   953  }
   954  
   955  func asString(b []byte) string {
   956  	return unsafe.String(unsafe.SliceData(b), len(b))
   957  }
   958  
   959  // A decoder helps decode the index format.
   960  type decoder struct {
   961  	data []byte // data after header
   962  	str  []byte // string table
   963  }
   964  
   965  // intAt returns the int at the given offset in d.data.
   966  func (d *decoder) intAt(off int) int {
   967  	if off < 0 || len(d.data)-off < 4 {
   968  		panic(errCorrupt)
   969  	}
   970  	i := binary.LittleEndian.Uint32(d.data[off : off+4])
   971  	if int32(i)>>31 != 0 {
   972  		panic(errCorrupt)
   973  	}
   974  	return int(i)
   975  }
   976  
   977  // boolAt returns the bool at the given offset in d.data.
   978  func (d *decoder) boolAt(off int) bool {
   979  	return d.intAt(off) != 0
   980  }
   981  
   982  // stringAt returns the string pointed at by the int at the given offset in d.data.
   983  func (d *decoder) stringAt(off int) string {
   984  	return d.stringTableAt(d.intAt(off))
   985  }
   986  
   987  // stringTableAt returns the string at the given offset in the string table d.str.
   988  func (d *decoder) stringTableAt(off int) string {
   989  	if off < 0 || off >= len(d.str) {
   990  		panic(errCorrupt)
   991  	}
   992  	s := d.str[off:]
   993  	v, n := binary.Uvarint(s)
   994  	if n <= 0 || v > uint64(len(s[n:])) {
   995  		panic(errCorrupt)
   996  	}
   997  	return asString(s[n : n+int(v)])
   998  }
   999  
  1000  // A reader reads sequential fields from a section of the index format.
  1001  type reader struct {
  1002  	d   *decoder
  1003  	pos int
  1004  }
  1005  
  1006  // readAt returns a reader starting at the given position in d.
  1007  func (d *decoder) readAt(pos int) *reader {
  1008  	return &reader{d, pos}
  1009  }
  1010  
  1011  // int reads the next int.
  1012  func (r *reader) int() int {
  1013  	i := r.d.intAt(r.pos)
  1014  	r.pos += 4
  1015  	return i
  1016  }
  1017  
  1018  // string reads the next string.
  1019  func (r *reader) string() string {
  1020  	return r.d.stringTableAt(r.int())
  1021  }
  1022  
  1023  // bool reads the next bool.
  1024  func (r *reader) bool() bool {
  1025  	return r.int() != 0
  1026  }
  1027  
  1028  // tokpos reads the next token.Position.
  1029  func (r *reader) tokpos() token.Position {
  1030  	return token.Position{
  1031  		Filename: r.string(),
  1032  		Offset:   r.int(),
  1033  		Line:     r.int(),
  1034  		Column:   r.int(),
  1035  	}
  1036  }
  1037  

View as plain text