example_test.go

Documentation: text/scanner

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package scanner_test
     6  
     7  import (
     8  	"fmt"
     9  	"strings"
    10  	"text/scanner"
    11  	"unicode"
    12  )
    13  
    14  func Example() {
    15  	const src = `
    16  // This is scanned code.
    17  if a > 10 {
    18  	someParsable = text
    19  }`
    20  
    21  	var s scanner.Scanner
    22  	s.Init(strings.NewReader(src))
    23  	s.Filename = "example"
    24  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    25  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
    26  	}
    27  
    28  	// Output:
    29  	// example:3:1: if
    30  	// example:3:4: a
    31  	// example:3:6: >
    32  	// example:3:8: 10
    33  	// example:3:11: {
    34  	// example:4:2: someParsable
    35  	// example:4:15: =
    36  	// example:4:17: text
    37  	// example:5:1: }
    38  }
    39  
    40  func Example_isIdentRune() {
    41  	const src = "%var1 var2%"
    42  
    43  	var s scanner.Scanner
    44  	s.Init(strings.NewReader(src))
    45  	s.Filename = "default"
    46  
    47  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    48  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
    49  	}
    50  
    51  	fmt.Println()
    52  	s.Init(strings.NewReader(src))
    53  	s.Filename = "percent"
    54  
    55  	// treat leading '%' as part of an identifier
    56  	s.IsIdentRune = func(ch rune, i int) bool {
    57  		return ch == '%' && i == 0 || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
    58  	}
    59  
    60  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    61  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
    62  	}
    63  
    64  	// Output:
    65  	// default:1:1: %
    66  	// default:1:2: var1
    67  	// default:1:7: var2
    68  	// default:1:11: %
    69  	//
    70  	// percent:1:1: %var1
    71  	// percent:1:7: var2
    72  	// percent:1:11: %
    73  }
    74  
    75  func Example_mode() {
    76  	const src = `
    77      // Comment begins at column 5.
    78  
    79  This line should not be included in the output.
    80  
    81  /*
    82  This multiline comment
    83  should be extracted in
    84  its entirety.
    85  */
    86  `
    87  
    88  	var s scanner.Scanner
    89  	s.Init(strings.NewReader(src))
    90  	s.Filename = "comments"
    91  	s.Mode ^= scanner.SkipComments // don't skip comments
    92  
    93  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    94  		txt := s.TokenText()
    95  		if strings.HasPrefix(txt, "//") || strings.HasPrefix(txt, "/*") {
    96  			fmt.Printf("%s: %s\n", s.Position, txt)
    97  		}
    98  	}
    99  
   100  	// Output:
   101  	// comments:2:5: // Comment begins at column 5.
   102  	// comments:6:1: /*
   103  	// This multiline comment
   104  	// should be extracted in
   105  	// its entirety.
   106  	// */
   107  }
   108  
   109  func Example_whitespace() {
   110  	// tab-separated values
   111  	const src = `aa	ab	ac	ad
   112  ba	bb	bc	bd
   113  ca	cb	cc	cd
   114  da	db	dc	dd`
   115  
   116  	var (
   117  		col, row int
   118  		s        scanner.Scanner
   119  		tsv      [4][4]string // large enough for example above
   120  	)
   121  	s.Init(strings.NewReader(src))
   122  	s.Whitespace ^= 1<<'\t' | 1<<'\n' // don't skip tabs and new lines
   123  
   124  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
   125  		switch tok {
   126  		case '\n':
   127  			row++
   128  			col = 0
   129  		case '\t':
   130  			col++
   131  		default:
   132  			tsv[row][col] = s.TokenText()
   133  		}
   134  	}
   135  
   136  	fmt.Print(tsv)
   137  
   138  	// Output:
   139  	// [[aa ab ac ad] [ba bb bc bd] [ca cb cc cd] [da db dc dd]]
   140  }
   141
View as plain text