token

Documentation

Index

• type Token
  • func Lex(input string) []Token
  • func (tok Token) String() string
• type TokenType
  • func (tt TokenType) IsOperator() bool
  • func (tt TokenType) String() string

Types

type Token

type Token struct {
	Type TokenType
	Text string
	Pos  int
}

func Lex

func Lex(input string) []Token

Lex the input into token tokens

func (Token) String

func (tok Token) String() string

type TokenType

type TokenType int

TokenType describes the type of token

const (
	// Special Tokens
	Illegal TokenType = iota
	EOF

	// Literal types
	Ident  // main
	Int    // 42
	Float  // 42.3
	String // 42

	// grouping, calls, indexing, fields and func args
	LParen // '('
	RParen // ')'
	LBrack // '['
	RBrack // ']'
	Dot    // '.'
	Comma  // ','

	Add // +
	Sub // -
	Mul // *
	Quo // /
	Rem // %

	Equal    // ==
	NotEqual // !=
	LTE      // <=
	GTE      // >=
	LT       // <
	GT       // >
	Not      // !

)

func (TokenType) IsOperator

func (tt TokenType) IsOperator() bool

func (TokenType) String

func (tt TokenType) String() string