lexer

Documentation

Index

• type Token
  • func Lex(input []rune, filename string, verbose bool) []*Token
• type TokenType
  • func (i TokenType) String() string

Types

type Token

type Token struct {
	Type                         TokenType
	Contents                     string
	CharNumber, LineNumber       int
	EndCharNumber, EndLineNumber int
	Filename                     string
}

func Lex

func Lex(input []rune, filename string, verbose bool) []*Token

type TokenType

type TokenType int

const (
	TOKEN_RUNE TokenType = iota
	TOKEN_IDENTIFIER
	TOKEN_SEPARATOR
	TOKEN_OPERATOR
	TOKEN_NUMBER
	TOKEN_ERRONEOUS
	TOKEN_STRING
	TOKEN_UNKNOWN
	TOKEN_COMMENT
	TOKEN_DOCCOMMENT
	TOKEN_END_OF_FILE
)

func (TokenType) String

func (i TokenType) String() string