lexer

Documentation

Index

• type Lexer
  • func NewLexer(s string, reservedIdentifiers map[string]TokenType, opts Options) (*Lexer, chan *Token)
  • func (l *Lexer) Run()
• type Options
• type Token
  • func MustMakeToken(value string) *Token
  • func (t Token) TokenType() TokenType
  • func (t Token) Value() string
• type TokenType
  • func (tt TokenType) String() string

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer breaks an input stream into a group of lexical elements.

func NewLexer

func NewLexer(s string, reservedIdentifiers map[string]TokenType, opts Options) (*Lexer, chan *Token)

NewLexer returns a lexer and an output channel for tokens.

func (*Lexer) Run

func (l *Lexer) Run()

Run consumes the input to produce a token stream.

type Options

type Options struct {
	EscapeAllNotOnlyQuotes bool
}

Options allows for specifying lexer options.

type Token

type Token struct {
	// contains filtered or unexported fields
}

Token is a token, doh!

func MustMakeToken

func MustMakeToken(value string) *Token

MustMakeToken is a test function for creating a Token.MustMakeToken.

func (Token) TokenType

func (t Token) TokenType() TokenType

TokenType returns the type of token consumed.

func (Token) Value

func (t Token) Value() string

Value returns the string representation of the token as needed.

type TokenType

type TokenType int

TokenType defines the type of identifier recognized by the Lexer.

const (
	// Error is what you get when the lexer fails to grok the input.
	Error TokenType = iota
	// Identifier is a symbol confining to C-style variable naming rules.
	Identifier
	// Pattern is a regex-ish pattern, accepts the following special chars: [{.*}].
	Pattern
	// Number is a numeral, including floats.
	Number
	// String is set of characters wrapped by double quotes.
	String
	// LParenthesis is the left parenthesis "(".
	LParenthesis
	// RParenthesis is the right parenthesis ")".
	RParenthesis
	// NotOperator is the exclamation sign - "!" symbol.
	NotOperator
	// Comma is a punctuation mark.
	Comma
	// Equal is the "=" symbol.
	Equal

	// True is Boolean true.
	True
	// False is Boolean false.
	False
)

func (TokenType) String

func (tt TokenType) String() string