tokenizer

Documentation

Index

• type TOKEN
  • func FromString(tk string) TOKEN
  • func (tk TOKEN) ToString() string
• type Token
  • func CreateToken(TYPE TOKEN, VALUE string, ORDER int) *Token
  • func (t *Token) GetOrder() int
  • func (t *Token) GetType() TOKEN
  • func (t *Token) GetValue() string
  • func (t *Token) SetOrder(ORDER int)

Types

type TOKEN

type TOKEN int

TOKEN - Custom type to hold value for token type

const (
	ADD            TOKEN = iota + 1 // EnumIndex = 1
	SUBTRACT                        // EnumIndex = 2
	DIVIDE                          // EnumIndex = 3
	MULTIPLY                        // EnumIndex = 4
	LPAREN                          // EnumIndex = 5
	RPAREN                          // EnumIndex = 6
	RBRACKET                        // EnumIndex = 7
	LBRACKET                        // EnumIndex = 8
	NUMBER                          // EnumIndex = 9
	IDENTIFIER                      // EnumIndex = 10
	TYPE                            // EnumIndex = 11
	COMMENT                         // EnumIndex = 12
	RSQUAREBRACKET                  // EnumIndex = 13
	LSQUAREBRACKET                  // EnumIndex = 14
	EQUAL                           // EnumIndex = 15
	STOP                            // EnumIndex = 16
	STRING                          // EnumIndex = 17
	NONE                            // EnumIndex = 18
	EXTENSION                       // EnumIndex = 19
	COMMA                           // EnumIndex = 20
)

Declare related constants for each Token type starting with index 1

func FromString

func FromString(tk string) TOKEN

func (TOKEN) ToString

func (tk TOKEN) ToString() string

Converting Enum to string

type Token

type Token struct {
	Type  TOKEN
	Value string
	Order int
}

func CreateToken

func CreateToken(TYPE TOKEN, VALUE string, ORDER int) *Token

Create Token object and return by pointer

func (*Token) GetOrder

func (t *Token) GetOrder() int

Order Getter

func (*Token) GetType

func (t *Token) GetType() TOKEN

Token Type Getter

func (*Token) GetValue

func (t *Token) GetValue() string

Value Getter

func (*Token) SetOrder

func (t *Token) SetOrder(ORDER int)

Set Order