generate.go

Documentation: cmd/go/internal/generate

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package generate implements the “go generate” command.
     6  package generate
     7  
     8  import (
     9  	"bufio"
    10  	"bytes"
    11  	"context"
    12  	"fmt"
    13  	"go/parser"
    14  	"go/token"
    15  	"io"
    16  	"log"
    17  	"os"
    18  	"os/exec"
    19  	"path/filepath"
    20  	"regexp"
    21  	"slices"
    22  	"strconv"
    23  	"strings"
    24  
    25  	"cmd/go/internal/base"
    26  	"cmd/go/internal/cfg"
    27  	"cmd/go/internal/load"
    28  	"cmd/go/internal/modload"
    29  	"cmd/go/internal/str"
    30  	"cmd/go/internal/work"
    31  )
    32  
    33  var CmdGenerate = &base.Command{
    34  	Run:       runGenerate,
    35  	UsageLine: "go generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]",
    36  	Short:     "generate Go files by processing source",
    37  	Long: `
    38  Generate runs commands described by directives within existing
    39  files. Those commands can run any process but the intent is to
    40  create or update Go source files.
    41  
    42  Go generate is never run automatically by go build, go test,
    43  and so on. It must be run explicitly.
    44  
    45  Go generate scans the file for directives, which are lines of
    46  the form,
    47  
    48  	//go:generate command argument...
    49  
    50  (note: no leading spaces and no space in "//go") where command
    51  is the generator to be run, corresponding to an executable file
    52  that can be run locally. It must either be in the shell path
    53  (gofmt), a fully qualified path (/usr/you/bin/mytool), or a
    54  command alias, described below.
    55  
    56  Note that go generate does not parse the file, so lines that look
    57  like directives in comments or multiline strings will be treated
    58  as directives.
    59  
    60  The arguments to the directive are space-separated tokens or
    61  double-quoted strings passed to the generator as individual
    62  arguments when it is run.
    63  
    64  Quoted strings use Go syntax and are evaluated before execution; a
    65  quoted string appears as a single argument to the generator.
    66  
    67  To convey to humans and machine tools that code is generated,
    68  generated source should have a line that matches the following
    69  regular expression (in Go syntax):
    70  
    71  	^// Code generated .* DO NOT EDIT\.$
    72  
    73  This line must appear before the first non-comment, non-blank
    74  text in the file.
    75  
    76  Go generate sets several variables when it runs the generator:
    77  
    78  	$GOARCH
    79  		The execution architecture (arm, amd64, etc.)
    80  	$GOOS
    81  		The execution operating system (linux, windows, etc.)
    82  	$GOFILE
    83  		The base name of the file.
    84  	$GOLINE
    85  		The line number of the directive in the source file.
    86  	$GOPACKAGE
    87  		The name of the package of the file containing the directive.
    88  	$GOROOT
    89  		The GOROOT directory for the 'go' command that invoked the
    90  		generator, containing the Go toolchain and standard library.
    91  	$DOLLAR
    92  		A dollar sign.
    93  	$PATH
    94  		The $PATH of the parent process, with $GOROOT/bin
    95  		placed at the beginning. This causes generators
    96  		that execute 'go' commands to use the same 'go'
    97  		as the parent 'go generate' command.
    98  
    99  Other than variable substitution and quoted-string evaluation, no
   100  special processing such as "globbing" is performed on the command
   101  line.
   102  
   103  As a last step before running the command, any invocations of any
   104  environment variables with alphanumeric names, such as $GOFILE or
   105  $HOME, are expanded throughout the command line. The syntax for
   106  variable expansion is $NAME on all operating systems. Due to the
   107  order of evaluation, variables are expanded even inside quoted
   108  strings. If the variable NAME is not set, $NAME expands to the
   109  empty string.
   110  
   111  A directive of the form,
   112  
   113  	//go:generate -command xxx args...
   114  
   115  specifies, for the remainder of this source file only, that the
   116  string xxx represents the command identified by the arguments. This
   117  can be used to create aliases or to handle multiword generators.
   118  For example,
   119  
   120  	//go:generate -command foo go tool foo
   121  
   122  specifies that the command "foo" represents the generator
   123  "go tool foo".
   124  
   125  Generate processes packages in the order given on the command line,
   126  one at a time. If the command line lists .go files from a single directory,
   127  they are treated as a single package. Within a package, generate processes the
   128  source files in a package in file name order, one at a time. Within
   129  a source file, generate runs generators in the order they appear
   130  in the file, one at a time. The go generate tool also sets the build
   131  tag "generate" so that files may be examined by go generate but ignored
   132  during build.
   133  
   134  For packages with invalid code, generate processes only source files with a
   135  valid package clause.
   136  
   137  If any generator returns an error exit status, "go generate" skips
   138  all further processing for that package.
   139  
   140  The generator is run in the package's source directory.
   141  
   142  Go generate accepts two specific flags:
   143  
   144  	-run=""
   145  		if non-empty, specifies a regular expression to select
   146  		directives whose full original source text (excluding
   147  		any trailing spaces and final newline) matches the
   148  		expression.
   149  
   150  	-skip=""
   151  		if non-empty, specifies a regular expression to suppress
   152  		directives whose full original source text (excluding
   153  		any trailing spaces and final newline) matches the
   154  		expression. If a directive matches both the -run and
   155  		the -skip arguments, it is skipped.
   156  
   157  It also accepts the standard build flags including -v, -n, and -x.
   158  The -v flag prints the names of packages and files as they are
   159  processed.
   160  The -n flag prints commands that would be executed.
   161  The -x flag prints commands as they are executed.
   162  
   163  For more about build flags, see 'go help build'.
   164  
   165  For more about specifying packages, see 'go help packages'.
   166  	`,
   167  }
   168  
   169  var (
   170  	generateRunFlag string         // generate -run flag
   171  	generateRunRE   *regexp.Regexp // compiled expression for -run
   172  
   173  	generateSkipFlag string         // generate -skip flag
   174  	generateSkipRE   *regexp.Regexp // compiled expression for -skip
   175  )
   176  
   177  func init() {
   178  	work.AddBuildFlags(CmdGenerate, work.DefaultBuildFlags)
   179  	CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
   180  	CmdGenerate.Flag.StringVar(&generateSkipFlag, "skip", "", "")
   181  }
   182  
   183  func runGenerate(ctx context.Context, cmd *base.Command, args []string) {
   184  	modload.InitWorkfile()
   185  
   186  	if generateRunFlag != "" {
   187  		var err error
   188  		generateRunRE, err = regexp.Compile(generateRunFlag)
   189  		if err != nil {
   190  			log.Fatalf("generate: %s", err)
   191  		}
   192  	}
   193  	if generateSkipFlag != "" {
   194  		var err error
   195  		generateSkipRE, err = regexp.Compile(generateSkipFlag)
   196  		if err != nil {
   197  			log.Fatalf("generate: %s", err)
   198  		}
   199  	}
   200  
   201  	cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "generate")
   202  
   203  	// Even if the arguments are .go files, this loop suffices.
   204  	printed := false
   205  	pkgOpts := load.PackageOpts{IgnoreImports: true}
   206  	for _, pkg := range load.PackagesAndErrors(ctx, pkgOpts, args) {
   207  		if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main {
   208  			if !printed {
   209  				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
   210  				printed = true
   211  			}
   212  			continue
   213  		}
   214  
   215  		if pkg.Error != nil && len(pkg.InternalAllGoFiles()) == 0 {
   216  			// A directory only contains a Go package if it has at least
   217  			// one .go source file, so the fact that there are no files
   218  			// implies that the package couldn't be found.
   219  			base.Errorf("%v", pkg.Error)
   220  		}
   221  
   222  		for _, file := range pkg.InternalGoFiles() {
   223  			if !generate(file) {
   224  				break
   225  			}
   226  		}
   227  
   228  		for _, file := range pkg.InternalXGoFiles() {
   229  			if !generate(file) {
   230  				break
   231  			}
   232  		}
   233  	}
   234  	base.ExitIfErrors()
   235  }
   236  
   237  // generate runs the generation directives for a single file.
   238  func generate(absFile string) bool {
   239  	src, err := os.ReadFile(absFile)
   240  	if err != nil {
   241  		log.Fatalf("generate: %s", err)
   242  	}
   243  
   244  	// Parse package clause
   245  	filePkg, err := parser.ParseFile(token.NewFileSet(), "", src, parser.PackageClauseOnly)
   246  	if err != nil {
   247  		// Invalid package clause - ignore file.
   248  		return true
   249  	}
   250  
   251  	g := &Generator{
   252  		r:        bytes.NewReader(src),
   253  		path:     absFile,
   254  		pkg:      filePkg.Name.String(),
   255  		commands: make(map[string][]string),
   256  	}
   257  	return g.run()
   258  }
   259  
   260  // A Generator represents the state of a single Go source file
   261  // being scanned for generator commands.
   262  type Generator struct {
   263  	r        io.Reader
   264  	path     string // full rooted path name.
   265  	dir      string // full rooted directory of file.
   266  	file     string // base name of file.
   267  	pkg      string
   268  	commands map[string][]string
   269  	lineNum  int // current line number.
   270  	env      []string
   271  }
   272  
   273  // run runs the generators in the current file.
   274  func (g *Generator) run() (ok bool) {
   275  	// Processing below here calls g.errorf on failure, which does panic(stop).
   276  	// If we encounter an error, we abort the package.
   277  	defer func() {
   278  		e := recover()
   279  		if e != nil {
   280  			ok = false
   281  			if e != stop {
   282  				panic(e)
   283  			}
   284  			base.SetExitStatus(1)
   285  		}
   286  	}()
   287  	g.dir, g.file = filepath.Split(g.path)
   288  	g.dir = filepath.Clean(g.dir) // No final separator please.
   289  	if cfg.BuildV {
   290  		fmt.Fprintf(os.Stderr, "%s\n", base.ShortPath(g.path))
   291  	}
   292  
   293  	// Scan for lines that start "//go:generate".
   294  	// Can't use bufio.Scanner because it can't handle long lines,
   295  	// which are likely to appear when using generate.
   296  	input := bufio.NewReader(g.r)
   297  	var err error
   298  	// One line per loop.
   299  	for {
   300  		g.lineNum++ // 1-indexed.
   301  		var buf []byte
   302  		buf, err = input.ReadSlice('\n')
   303  		if err == bufio.ErrBufferFull {
   304  			// Line too long - consume and ignore.
   305  			if isGoGenerate(buf) {
   306  				g.errorf("directive too long")
   307  			}
   308  			for err == bufio.ErrBufferFull {
   309  				_, err = input.ReadSlice('\n')
   310  			}
   311  			if err != nil {
   312  				break
   313  			}
   314  			continue
   315  		}
   316  
   317  		if err != nil {
   318  			// Check for marker at EOF without final \n.
   319  			if err == io.EOF && isGoGenerate(buf) {
   320  				err = io.ErrUnexpectedEOF
   321  			}
   322  			break
   323  		}
   324  
   325  		if !isGoGenerate(buf) {
   326  			continue
   327  		}
   328  		if generateRunFlag != "" && !generateRunRE.Match(bytes.TrimSpace(buf)) {
   329  			continue
   330  		}
   331  		if generateSkipFlag != "" && generateSkipRE.Match(bytes.TrimSpace(buf)) {
   332  			continue
   333  		}
   334  
   335  		g.setEnv()
   336  		words := g.split(string(buf))
   337  		if len(words) == 0 {
   338  			g.errorf("no arguments to directive")
   339  		}
   340  		if words[0] == "-command" {
   341  			g.setShorthand(words)
   342  			continue
   343  		}
   344  		// Run the command line.
   345  		if cfg.BuildN || cfg.BuildX {
   346  			fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " "))
   347  		}
   348  		if cfg.BuildN {
   349  			continue
   350  		}
   351  		g.exec(words)
   352  	}
   353  	if err != nil && err != io.EOF {
   354  		g.errorf("error reading %s: %s", base.ShortPath(g.path), err)
   355  	}
   356  	return true
   357  }
   358  
   359  func isGoGenerate(buf []byte) bool {
   360  	return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t"))
   361  }
   362  
   363  // setEnv sets the extra environment variables used when executing a
   364  // single go:generate command.
   365  func (g *Generator) setEnv() {
   366  	env := []string{
   367  		"GOROOT=" + cfg.GOROOT,
   368  		"GOARCH=" + cfg.BuildContext.GOARCH,
   369  		"GOOS=" + cfg.BuildContext.GOOS,
   370  		"GOFILE=" + g.file,
   371  		"GOLINE=" + strconv.Itoa(g.lineNum),
   372  		"GOPACKAGE=" + g.pkg,
   373  		"DOLLAR=" + "$",
   374  	}
   375  	env = base.AppendPATH(env)
   376  	env = base.AppendPWD(env, g.dir)
   377  	g.env = env
   378  }
   379  
   380  // split breaks the line into words, evaluating quoted
   381  // strings and evaluating environment variables.
   382  // The initial //go:generate element is present in line.
   383  func (g *Generator) split(line string) []string {
   384  	// Parse line, obeying quoted strings.
   385  	var words []string
   386  	line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline.
   387  	// There may still be a carriage return.
   388  	if len(line) > 0 && line[len(line)-1] == '\r' {
   389  		line = line[:len(line)-1]
   390  	}
   391  	// One (possibly quoted) word per iteration.
   392  Words:
   393  	for {
   394  		line = strings.TrimLeft(line, " \t")
   395  		if len(line) == 0 {
   396  			break
   397  		}
   398  		if line[0] == '"' {
   399  			for i := 1; i < len(line); i++ {
   400  				c := line[i] // Only looking for ASCII so this is OK.
   401  				switch c {
   402  				case '\\':
   403  					if i+1 == len(line) {
   404  						g.errorf("bad backslash")
   405  					}
   406  					i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
   407  				case '"':
   408  					word, err := strconv.Unquote(line[0 : i+1])
   409  					if err != nil {
   410  						g.errorf("bad quoted string")
   411  					}
   412  					words = append(words, word)
   413  					line = line[i+1:]
   414  					// Check the next character is space or end of line.
   415  					if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
   416  						g.errorf("expect space after quoted argument")
   417  					}
   418  					continue Words
   419  				}
   420  			}
   421  			g.errorf("mismatched quoted string")
   422  		}
   423  		i := strings.IndexAny(line, " \t")
   424  		if i < 0 {
   425  			i = len(line)
   426  		}
   427  		words = append(words, line[0:i])
   428  		line = line[i:]
   429  	}
   430  	// Substitute command if required.
   431  	if len(words) > 0 && g.commands[words[0]] != nil {
   432  		// Replace 0th word by command substitution.
   433  		//
   434  		// Force a copy of the command definition to
   435  		// ensure words doesn't end up as a reference
   436  		// to the g.commands content.
   437  		tmpCmdWords := append([]string(nil), (g.commands[words[0]])...)
   438  		words = append(tmpCmdWords, words[1:]...)
   439  	}
   440  	// Substitute environment variables.
   441  	for i, word := range words {
   442  		words[i] = os.Expand(word, g.expandVar)
   443  	}
   444  	return words
   445  }
   446  
   447  var stop = fmt.Errorf("error in generation")
   448  
   449  // errorf logs an error message prefixed with the file and line number.
   450  // It then exits the program (with exit status 1) because generation stops
   451  // at the first error.
   452  func (g *Generator) errorf(format string, args ...any) {
   453  	fmt.Fprintf(os.Stderr, "%s:%d: %s\n", base.ShortPath(g.path), g.lineNum,
   454  		fmt.Sprintf(format, args...))
   455  	panic(stop)
   456  }
   457  
   458  // expandVar expands the $XXX invocation in word. It is called
   459  // by os.Expand.
   460  func (g *Generator) expandVar(word string) string {
   461  	w := word + "="
   462  	for _, e := range g.env {
   463  		if strings.HasPrefix(e, w) {
   464  			return e[len(w):]
   465  		}
   466  	}
   467  	return os.Getenv(word)
   468  }
   469  
   470  // setShorthand installs a new shorthand as defined by a -command directive.
   471  func (g *Generator) setShorthand(words []string) {
   472  	// Create command shorthand.
   473  	if len(words) == 1 {
   474  		g.errorf("no command specified for -command")
   475  	}
   476  	command := words[1]
   477  	if g.commands[command] != nil {
   478  		g.errorf("command %q multiply defined", command)
   479  	}
   480  	g.commands[command] = slices.Clip(words[2:])
   481  }
   482  
   483  // exec runs the command specified by the argument. The first word is
   484  // the command name itself.
   485  func (g *Generator) exec(words []string) {
   486  	path := words[0]
   487  	if path != "" && !strings.Contains(path, string(os.PathSeparator)) {
   488  		// If a generator says '//go:generate go run <blah>' it almost certainly
   489  		// intends to use the same 'go' as 'go generate' itself.
   490  		// Prefer to resolve the binary from GOROOT/bin, and for consistency
   491  		// prefer to resolve any other commands there too.
   492  		gorootBinPath, err := cfg.LookPath(filepath.Join(cfg.GOROOTbin, path))
   493  		if err == nil {
   494  			path = gorootBinPath
   495  		}
   496  	}
   497  	cmd := exec.Command(path, words[1:]...)
   498  	cmd.Args[0] = words[0] // Overwrite with the original in case it was rewritten above.
   499  
   500  	// Standard in and out of generator should be the usual.
   501  	cmd.Stdout = os.Stdout
   502  	cmd.Stderr = os.Stderr
   503  	// Run the command in the package directory.
   504  	cmd.Dir = g.dir
   505  	cmd.Env = str.StringList(cfg.OrigEnv, g.env)
   506  	err := cmd.Run()
   507  	if err != nil {
   508  		g.errorf("running %q: %s", words[0], err)
   509  	}
   510  }
   511
View as plain text