tokenizer

Documentation

Index

• type Token
  • func NewToken(tp, text string) *Token
  • func Tokenize(text string) []*Token
• type TokenType

Types

type Token

type Token struct {
	Type  TokenType
	Value string
}

func NewToken

func NewToken(tp, text string) *Token

func Tokenize

func Tokenize(text string) []*Token

type TokenType

type TokenType = string

const (
	Underline          TokenType = "_"
	Star               TokenType = "*"
	Hash               TokenType = "#"
	Backtick           TokenType = "`"
	LeftSquareBracket  TokenType = "["
	RightSquareBracket TokenType = "]"
	LeftParenthesis    TokenType = "("
	RightParenthesis   TokenType = ")"
	ExclamationMark    TokenType = "!"
	Newline            TokenType = "\n"
	Space              TokenType = " "
)

const (
	Text TokenType = ""
)