lexer

README

go-lexer

Implementation of a basic lexer from Rob Pike talk https://www.youtube.com/watch?v=HxaD_trXwRE
Used https://github.com/bbuck/go-lexer as reference

Documentation

Index

• Constants
• type Lexer
  • func New(name, s string, startState StateFn) Lexer
• type StateFn
• type Token
• type TokenType

Constants

const (
	EOFRune    rune      = -1
	ErrorToken TokenType = 0
)

Types

type Lexer

type Lexer interface {
	NextToken() (Token, bool)
	Start()
	Emit(t TokenType)
	Next() (r rune)
	Peek() (r rune)
	Ignore()
	Backup()
	Accept(valid string) bool
	AcceptRun(valid string)
	Errorf(format string, args ...interface{}) StateFn
}

func New

func New(name, s string, startState StateFn) Lexer

type StateFn

type StateFn func(Lexer) StateFn

type Token

type Token interface {
	Type() TokenType
	At() int
	Value() string
}

type TokenType

type TokenType int