lexer

Documentation

Index

• Constants
• type Lexeme
• type Lexer
  • func Lex(source input.InputSource, input string) *Lexer
• type PeekableLexer
  • func NewPeekableLexer(lex *Lexer) *PeekableLexer
  • func (l *PeekableLexer) NextToken() Lexeme
  • func (l *PeekableLexer) PeekToken(count int) Lexeme
• type TokenType
  • func (i TokenType) String() string

Constants

const EOFRUNE = -1

Types

type Lexeme

type Lexeme struct {
	Kind     TokenType          // The type of this lexeme.
	Position input.BytePosition // The starting position of this token in the input string.
	Value    string             // The textual value of this token.
}

Lexeme represents a token returned from scanning the contents of a file.

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer holds the state of the scanner.

func Lex

func Lex(source input.InputSource, input string) *Lexer

Lex creates a new scanner for the input string.

type PeekableLexer

type PeekableLexer struct {
	// contains filtered or unexported fields
}

PeekableLexer wraps a lexer and provides the ability to peek forward without losing state.

func NewPeekableLexer

func NewPeekableLexer(lex *Lexer) *PeekableLexer

NewPeekableLexer returns a new PeekableLexer for the given lexer.

func (*PeekableLexer) NextToken

func (l *PeekableLexer) NextToken() Lexeme

NextToken returns the next token found in the lexer.

func (*PeekableLexer) PeekToken

func (l *PeekableLexer) PeekToken(count int) Lexeme

PeekToken performs lookahead of the given count on the token stream.

type TokenType

type TokenType int

TokenType identifies the type of lexer lexemes.

const (
	TokenTypeError TokenType = iota // error occurred; value is text of error

	// Synthetic semicolon
	TokenTypeSyntheticSemicolon

	TokenTypeEOF
	TokenTypeWhitespace
	TokenTypeSinglelineComment
	TokenTypeMultilineComment
	TokenTypeNewline

	TokenTypeKeyword    // interface
	TokenTypeIdentifier // helloworld
	TokenTypeNumber     // 123

	TokenTypeLeftBrace  // {
	TokenTypeRightBrace // }
	TokenTypeLeftParen  // (
	TokenTypeRightParen // )

	TokenTypePipe  // |
	TokenTypePlus  // +
	TokenTypeMinus // -
	TokenTypeAnd   // &
	TokenTypeDiv   // /

	TokenTypeEquals     // =
	TokenTypeColon      // :
	TokenTypeSemicolon  // ;
	TokenTypeRightArrow // ->
	TokenTypeHash       // #
	TokenTypeEllipsis   // ...
)

func (TokenType) String

func (i TokenType) String() string