input.go

Documentation: cmd/asm/internal/lex

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package lex
     6  
     7  import (
     8  	"fmt"
     9  	"os"
    10  	"path/filepath"
    11  	"strconv"
    12  	"strings"
    13  	"text/scanner"
    14  
    15  	"cmd/asm/internal/flags"
    16  	"cmd/internal/objabi"
    17  	"cmd/internal/src"
    18  )
    19  
    20  // Input is the main input: a stack of readers and some macro definitions.
    21  // It also handles #include processing (by pushing onto the input stack)
    22  // and parses and instantiates macro definitions.
    23  type Input struct {
    24  	Stack
    25  	includes        []string
    26  	beginningOfLine bool
    27  	ifdefStack      []bool
    28  	macros          map[string]*Macro
    29  	text            string // Text of last token returned by Next.
    30  	peek            bool
    31  	peekToken       ScanToken
    32  	peekText        string
    33  }
    34  
    35  // NewInput returns an Input from the given path.
    36  func NewInput(name string) *Input {
    37  	return &Input{
    38  		// include directories: look in source dir, then -I directories.
    39  		includes:        append([]string{filepath.Dir(name)}, flags.I...),
    40  		beginningOfLine: true,
    41  		macros:          predefine(flags.D),
    42  	}
    43  }
    44  
    45  // predefine installs the macros set by the -D flag on the command line.
    46  func predefine(defines flags.MultiFlag) map[string]*Macro {
    47  	macros := make(map[string]*Macro)
    48  	for _, name := range defines {
    49  		value := "1"
    50  		i := strings.IndexRune(name, '=')
    51  		if i > 0 {
    52  			name, value = name[:i], name[i+1:]
    53  		}
    54  		tokens := Tokenize(name)
    55  		if len(tokens) != 1 || tokens[0].ScanToken != scanner.Ident {
    56  			fmt.Fprintf(os.Stderr, "asm: parsing -D: %q is not a valid identifier name\n", tokens[0])
    57  			flags.Usage()
    58  		}
    59  		macros[name] = &Macro{
    60  			name:   name,
    61  			args:   nil,
    62  			tokens: Tokenize(value),
    63  		}
    64  	}
    65  	return macros
    66  }
    67  
    68  var panicOnError bool // For testing.
    69  
    70  func (in *Input) Error(args ...interface{}) {
    71  	if panicOnError {
    72  		panic(fmt.Errorf("%s:%d: %s", in.File(), in.Line(), fmt.Sprintln(args...)))
    73  	}
    74  	fmt.Fprintf(os.Stderr, "%s:%d: %s", in.File(), in.Line(), fmt.Sprintln(args...))
    75  	os.Exit(1)
    76  }
    77  
    78  // expectText is like Error but adds "got XXX" where XXX is a quoted representation of the most recent token.
    79  func (in *Input) expectText(args ...interface{}) {
    80  	in.Error(append(args, "; got", strconv.Quote(in.Stack.Text()))...)
    81  }
    82  
    83  // enabled reports whether the input is enabled by an ifdef, or is at the top level.
    84  func (in *Input) enabled() bool {
    85  	return len(in.ifdefStack) == 0 || in.ifdefStack[len(in.ifdefStack)-1]
    86  }
    87  
    88  func (in *Input) expectNewline(directive string) {
    89  	tok := in.Stack.Next()
    90  	if tok != '\n' {
    91  		in.expectText("expected newline after", directive)
    92  	}
    93  }
    94  
    95  func (in *Input) Next() ScanToken {
    96  	if in.peek {
    97  		in.peek = false
    98  		tok := in.peekToken
    99  		in.text = in.peekText
   100  		return tok
   101  	}
   102  	// If we cannot generate a token after 100 macro invocations, we're in trouble.
   103  	// The usual case is caught by Push, below, but be safe.
   104  	for nesting := 0; nesting < 100; {
   105  		tok := in.Stack.Next()
   106  		switch tok {
   107  		case '#':
   108  			if !in.beginningOfLine {
   109  				in.Error("'#' must be first item on line")
   110  			}
   111  			in.beginningOfLine = in.hash()
   112  			in.text = "#"
   113  			return '#'
   114  
   115  		case scanner.Ident:
   116  			// Is it a macro name?
   117  			name := in.Stack.Text()
   118  			macro := in.macros[name]
   119  			if macro != nil {
   120  				nesting++
   121  				in.invokeMacro(macro)
   122  				continue
   123  			}
   124  			fallthrough
   125  		default:
   126  			if tok == scanner.EOF && len(in.ifdefStack) > 0 {
   127  				// We're skipping text but have run out of input with no #endif.
   128  				in.Error("unclosed #ifdef or #ifndef")
   129  			}
   130  			in.beginningOfLine = tok == '\n'
   131  			if in.enabled() {
   132  				in.text = in.Stack.Text()
   133  				return tok
   134  			}
   135  		}
   136  	}
   137  	in.Error("recursive macro invocation")
   138  	return 0
   139  }
   140  
   141  func (in *Input) Text() string {
   142  	return in.text
   143  }
   144  
   145  // hash processes a # preprocessor directive. It reports whether it completes.
   146  func (in *Input) hash() bool {
   147  	// We have a '#'; it must be followed by a known word (define, include, etc.).
   148  	tok := in.Stack.Next()
   149  	if tok != scanner.Ident {
   150  		in.expectText("expected identifier after '#'")
   151  	}
   152  	if !in.enabled() {
   153  		// Can only start including again if we are at #else or #endif but also
   154  		// need to keep track of nested #if[n]defs.
   155  		// We let #line through because it might affect errors.
   156  		switch in.Stack.Text() {
   157  		case "else", "endif", "ifdef", "ifndef", "line":
   158  			// Press on.
   159  		default:
   160  			return false
   161  		}
   162  	}
   163  	switch in.Stack.Text() {
   164  	case "define":
   165  		in.define()
   166  	case "else":
   167  		in.else_()
   168  	case "endif":
   169  		in.endif()
   170  	case "ifdef":
   171  		in.ifdef(true)
   172  	case "ifndef":
   173  		in.ifdef(false)
   174  	case "include":
   175  		in.include()
   176  	case "line":
   177  		in.line()
   178  	case "undef":
   179  		in.undef()
   180  	default:
   181  		in.Error("unexpected token after '#':", in.Stack.Text())
   182  	}
   183  	return true
   184  }
   185  
   186  // macroName returns the name for the macro being referenced.
   187  func (in *Input) macroName() string {
   188  	// We use the Stack's input method; no macro processing at this stage.
   189  	tok := in.Stack.Next()
   190  	if tok != scanner.Ident {
   191  		in.expectText("expected identifier after # directive")
   192  	}
   193  	// Name is alphanumeric by definition.
   194  	return in.Stack.Text()
   195  }
   196  
   197  // #define processing.
   198  func (in *Input) define() {
   199  	name := in.macroName()
   200  	args, tokens := in.macroDefinition(name)
   201  	in.defineMacro(name, args, tokens)
   202  }
   203  
   204  // defineMacro stores the macro definition in the Input.
   205  func (in *Input) defineMacro(name string, args []string, tokens []Token) {
   206  	if in.macros[name] != nil {
   207  		in.Error("redefinition of macro:", name)
   208  	}
   209  	in.macros[name] = &Macro{
   210  		name:   name,
   211  		args:   args,
   212  		tokens: tokens,
   213  	}
   214  }
   215  
   216  // macroDefinition returns the list of formals and the tokens of the definition.
   217  // The argument list is nil for no parens on the definition; otherwise a list of
   218  // formal argument names.
   219  func (in *Input) macroDefinition(name string) ([]string, []Token) {
   220  	prevCol := in.Stack.Col()
   221  	tok := in.Stack.Next()
   222  	if tok == '\n' || tok == scanner.EOF {
   223  		return nil, nil // No definition for macro
   224  	}
   225  	var args []string
   226  	// The C preprocessor treats
   227  	//	#define A(x)
   228  	// and
   229  	//	#define A (x)
   230  	// distinctly: the first is a macro with arguments, the second without.
   231  	// Distinguish these cases using the column number, since we don't
   232  	// see the space itself. Note that text/scanner reports the position at the
   233  	// end of the token. It's where you are now, and you just read this token.
   234  	if tok == '(' && in.Stack.Col() == prevCol+1 {
   235  		// Macro has arguments. Scan list of formals.
   236  		acceptArg := true
   237  		args = []string{} // Zero length but not nil.
   238  	Loop:
   239  		for {
   240  			tok = in.Stack.Next()
   241  			switch tok {
   242  			case ')':
   243  				tok = in.Stack.Next() // First token of macro definition.
   244  				break Loop
   245  			case ',':
   246  				if acceptArg {
   247  					in.Error("bad syntax in definition for macro:", name)
   248  				}
   249  				acceptArg = true
   250  			case scanner.Ident:
   251  				if !acceptArg {
   252  					in.Error("bad syntax in definition for macro:", name)
   253  				}
   254  				arg := in.Stack.Text()
   255  				if i := lookup(args, arg); i >= 0 {
   256  					in.Error("duplicate argument", arg, "in definition for macro:", name)
   257  				}
   258  				args = append(args, arg)
   259  				acceptArg = false
   260  			default:
   261  				in.Error("bad definition for macro:", name)
   262  			}
   263  		}
   264  	}
   265  	var tokens []Token
   266  	// Scan to newline. Backslashes escape newlines.
   267  	for tok != '\n' {
   268  		if tok == scanner.EOF {
   269  			in.Error("missing newline in definition for macro:", name)
   270  		}
   271  		if tok == '\\' {
   272  			tok = in.Stack.Next()
   273  			if tok != '\n' && tok != '\\' {
   274  				in.Error(`can only escape \ or \n in definition for macro:`, name)
   275  			}
   276  		}
   277  		tokens = append(tokens, Make(tok, in.Stack.Text()))
   278  		tok = in.Stack.Next()
   279  	}
   280  	return args, tokens
   281  }
   282  
   283  func lookup(args []string, arg string) int {
   284  	for i, a := range args {
   285  		if a == arg {
   286  			return i
   287  		}
   288  	}
   289  	return -1
   290  }
   291  
   292  // invokeMacro pushes onto the input Stack a Slice that holds the macro definition with the actual
   293  // parameters substituted for the formals.
   294  // Invoking a macro does not touch the PC/line history.
   295  func (in *Input) invokeMacro(macro *Macro) {
   296  	// If the macro has no arguments, just substitute the text.
   297  	if macro.args == nil {
   298  		in.Push(NewSlice(in.Base(), in.Line(), macro.tokens))
   299  		return
   300  	}
   301  	tok := in.Stack.Next()
   302  	if tok != '(' {
   303  		// If the macro has arguments but is invoked without them, all we push is the macro name.
   304  		// First, put back the token.
   305  		in.peekToken = tok
   306  		in.peekText = in.text
   307  		in.peek = true
   308  		in.Push(NewSlice(in.Base(), in.Line(), []Token{Make(macroName, macro.name)}))
   309  		return
   310  	}
   311  	actuals := in.argsFor(macro)
   312  	var tokens []Token
   313  	for _, tok := range macro.tokens {
   314  		if tok.ScanToken != scanner.Ident {
   315  			tokens = append(tokens, tok)
   316  			continue
   317  		}
   318  		substitution := actuals[tok.text]
   319  		if substitution == nil {
   320  			tokens = append(tokens, tok)
   321  			continue
   322  		}
   323  		tokens = append(tokens, substitution...)
   324  	}
   325  	in.Push(NewSlice(in.Base(), in.Line(), tokens))
   326  }
   327  
   328  // argsFor returns a map from formal name to actual value for this argumented macro invocation.
   329  // The opening parenthesis has been absorbed.
   330  func (in *Input) argsFor(macro *Macro) map[string][]Token {
   331  	var args [][]Token
   332  	// One macro argument per iteration. Collect them all and check counts afterwards.
   333  	for argNum := 0; ; argNum++ {
   334  		tokens, tok := in.collectArgument(macro)
   335  		args = append(args, tokens)
   336  		if tok == ')' {
   337  			break
   338  		}
   339  	}
   340  	// Zero-argument macros are tricky.
   341  	if len(macro.args) == 0 && len(args) == 1 && args[0] == nil {
   342  		args = nil
   343  	} else if len(args) != len(macro.args) {
   344  		in.Error("wrong arg count for macro", macro.name)
   345  	}
   346  	argMap := make(map[string][]Token)
   347  	for i, arg := range args {
   348  		argMap[macro.args[i]] = arg
   349  	}
   350  	return argMap
   351  }
   352  
   353  // collectArgument returns the actual tokens for a single argument of a macro.
   354  // It also returns the token that terminated the argument, which will always
   355  // be either ',' or ')'. The starting '(' has been scanned.
   356  func (in *Input) collectArgument(macro *Macro) ([]Token, ScanToken) {
   357  	nesting := 0
   358  	var tokens []Token
   359  	for {
   360  		tok := in.Stack.Next()
   361  		if tok == scanner.EOF || tok == '\n' {
   362  			in.Error("unterminated arg list invoking macro:", macro.name)
   363  		}
   364  		if nesting == 0 && (tok == ')' || tok == ',') {
   365  			return tokens, tok
   366  		}
   367  		if tok == '(' {
   368  			nesting++
   369  		}
   370  		if tok == ')' {
   371  			nesting--
   372  		}
   373  		tokens = append(tokens, Make(tok, in.Stack.Text()))
   374  	}
   375  }
   376  
   377  // #ifdef and #ifndef processing.
   378  func (in *Input) ifdef(truth bool) {
   379  	name := in.macroName()
   380  	in.expectNewline("#if[n]def")
   381  	if !in.enabled() {
   382  		truth = false
   383  	} else if _, defined := in.macros[name]; !defined {
   384  		truth = !truth
   385  	}
   386  	in.ifdefStack = append(in.ifdefStack, truth)
   387  }
   388  
   389  // #else processing
   390  func (in *Input) else_() {
   391  	in.expectNewline("#else")
   392  	if len(in.ifdefStack) == 0 {
   393  		in.Error("unmatched #else")
   394  	}
   395  	if len(in.ifdefStack) == 1 || in.ifdefStack[len(in.ifdefStack)-2] {
   396  		in.ifdefStack[len(in.ifdefStack)-1] = !in.ifdefStack[len(in.ifdefStack)-1]
   397  	}
   398  }
   399  
   400  // #endif processing.
   401  func (in *Input) endif() {
   402  	in.expectNewline("#endif")
   403  	if len(in.ifdefStack) == 0 {
   404  		in.Error("unmatched #endif")
   405  	}
   406  	in.ifdefStack = in.ifdefStack[:len(in.ifdefStack)-1]
   407  }
   408  
   409  // #include processing.
   410  func (in *Input) include() {
   411  	// Find and parse string.
   412  	tok := in.Stack.Next()
   413  	if tok != scanner.String {
   414  		in.expectText("expected string after #include")
   415  	}
   416  	name, err := strconv.Unquote(in.Stack.Text())
   417  	if err != nil {
   418  		in.Error("unquoting include file name: ", err)
   419  	}
   420  	in.expectNewline("#include")
   421  	// Push tokenizer for file onto stack.
   422  	fd, err := os.Open(name)
   423  	if err != nil {
   424  		for _, dir := range in.includes {
   425  			fd, err = os.Open(filepath.Join(dir, name))
   426  			if err == nil {
   427  				break
   428  			}
   429  		}
   430  		if err != nil {
   431  			in.Error("#include:", err)
   432  		}
   433  	}
   434  	in.Push(NewTokenizer(name, fd, fd))
   435  }
   436  
   437  // #line processing.
   438  func (in *Input) line() {
   439  	// Only need to handle Plan 9 format: #line 337 "filename"
   440  	tok := in.Stack.Next()
   441  	if tok != scanner.Int {
   442  		in.expectText("expected line number after #line")
   443  	}
   444  	line, err := strconv.Atoi(in.Stack.Text())
   445  	if err != nil {
   446  		in.Error("error parsing #line (cannot happen):", err)
   447  	}
   448  	tok = in.Stack.Next()
   449  	if tok != scanner.String {
   450  		in.expectText("expected file name in #line")
   451  	}
   452  	file, err := strconv.Unquote(in.Stack.Text())
   453  	if err != nil {
   454  		in.Error("unquoting #line file name: ", err)
   455  	}
   456  	tok = in.Stack.Next()
   457  	if tok != '\n' {
   458  		in.Error("unexpected token at end of #line: ", tok)
   459  	}
   460  	pos := src.MakePos(in.Base(), uint(in.Line())+1, 1) // +1 because #line nnn means line nnn starts on next line
   461  	in.Stack.SetBase(src.NewLinePragmaBase(pos, file, objabi.AbsFile(objabi.WorkingDir(), file, *flags.TrimPath), uint(line), 1))
   462  }
   463  
   464  // #undef processing
   465  func (in *Input) undef() {
   466  	name := in.macroName()
   467  	if in.macros[name] == nil {
   468  		in.Error("#undef for undefined macro:", name)
   469  	}
   470  	// Newline must be next.
   471  	tok := in.Stack.Next()
   472  	if tok != '\n' {
   473  		in.Error("syntax error in #undef for macro:", name)
   474  	}
   475  	delete(in.macros, name)
   476  }
   477  
   478  func (in *Input) Push(r TokenReader) {
   479  	if len(in.tr) > 100 {
   480  		in.Error("input recursion")
   481  	}
   482  	in.Stack.Push(r)
   483  }
   484  
   485  func (in *Input) Close() {
   486  }
   487
View as plain text