lexer

Documentation

Index

• Variables
• type BaseLexer
  • func NewBaseLexer() BaseLexer
  • func (lex *BaseLexer) SetData(src string)
  • func (lex *BaseLexer) Tz() *Tokenizer
• type Lexer
  • func NewLexer() *Lexer
  • func (lex *Lexer) AddDefaultMatcher(m TokenMatcher)
  • func (lex *Lexer) AddIgnoreMatcher(m TokenMatcher)
  • func (lex *Lexer) AddMatcher(m TokenMatcher)
  • func (lex *Lexer) Error(err string)
  • func (lex *Lexer) Lex() Token
  • func (lex *Lexer) SetData(src string)
• type Token
  • func NewToken(m TokenMatcher, tz *Tokenizer, v string, raw string) Token
• type TokenMatcher
  • func NewBackTicksMatcher(id int) TokenMatcher
  • func NewCStyleCommentMatcher(id int) TokenMatcher
  • func NewCStyleMulCommentMatcher(id int) TokenMatcher
  • func NewIdentifierMatcher(id int) TokenMatcher
  • func NewKeywordMatcher(id int, word string) TokenMatcher
  • func NewLineEndMatcher(id int) TokenMatcher
  • func NewNumeralMatcher(id int) TokenMatcher
  • func NewPositiveNumeralMatcher(id int) TokenMatcher
  • func NewSignMatcher(word string) TokenMatcher
  • func NewStringMatcher(id int) TokenMatcher
  • func NewUnixStyleCommentMatcher(id int) TokenMatcher
  • func NewUnknownMatcher(id int) TokenMatcher
  • func NewWhiteSpaceMatcher(id int) TokenMatcher
  • func NewjavaStyleDocMatcher(id int) TokenMatcher
• type TokenPos
  • func (self TokenPos) String() string
• type Tokenizer
  • func NewTokenizer(s string, l *BaseLexer) *Tokenizer
  • func (self *Tokenizer) ConsumeMulti(count int)
  • func (self *Tokenizer) ConsumeOne()
  • func (self *Tokenizer) Count() int
  • func (self *Tokenizer) Current() rune
  • func (self *Tokenizer) EOF() bool
  • func (self *Tokenizer) Index() int
  • func (self *Tokenizer) Line() int
  • func (self *Tokenizer) Peek(offset int) rune
  • func (self *Tokenizer) StringRange(begin, end int) string

Variables

var DefaultTokenPos = TokenPos{Line: 1, Col: 1}

Types

type BaseLexer

type BaseLexer struct {
	// contains filtered or unexported fields
}

func NewBaseLexer

func NewBaseLexer() BaseLexer

func (*BaseLexer) SetData

func (lex *BaseLexer) SetData(src string)

func (*BaseLexer) Tz

func (lex *BaseLexer) Tz() *Tokenizer

type Lexer

type Lexer struct {
	BaseLexer
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer() *Lexer

func (*Lexer) AddDefaultMatcher

func (lex *Lexer) AddDefaultMatcher(m TokenMatcher)

func (*Lexer) AddIgnoreMatcher

func (lex *Lexer) AddIgnoreMatcher(m TokenMatcher)

func (*Lexer) AddMatcher

func (lex *Lexer) AddMatcher(m TokenMatcher)

func (*Lexer) Error

func (lex *Lexer) Error(err string)

func (*Lexer) Lex

func (lex *Lexer) Lex() Token

func (*Lexer) SetData

func (lex *Lexer) SetData(src string)

type Token

type Token interface {
	Line() int
	Index() int
	MatcherID() int
	Value() string
	Raw() string
	ToFloat32() float32
	ToInt32() int32
	ToInt() int
	ToInt64() int64
	MatcherName() string
	MatcherString() string
	String() string
}

var EmptyToken Token

func NewToken

func NewToken(m TokenMatcher, tz *Tokenizer, v string, raw string) Token

type TokenMatcher

type TokenMatcher interface {
	Match(*Tokenizer) (Token, error)
	ID() int
	String() string
}

func NewBackTicksMatcher

func NewBackTicksMatcher(id int) TokenMatcher

func NewCStyleCommentMatcher

func NewCStyleCommentMatcher(id int) TokenMatcher

func NewCStyleMulCommentMatcher

func NewCStyleMulCommentMatcher(id int) TokenMatcher

func NewIdentifierMatcher

func NewIdentifierMatcher(id int) TokenMatcher

func NewKeywordMatcher

func NewKeywordMatcher(id int, word string) TokenMatcher

func NewLineEndMatcher

func NewLineEndMatcher(id int) TokenMatcher

func NewNumeralMatcher

func NewNumeralMatcher(id int) TokenMatcher

func NewPositiveNumeralMatcher

func NewPositiveNumeralMatcher(id int) TokenMatcher

func NewSignMatcher

func NewSignMatcher(word string) TokenMatcher

func NewStringMatcher

func NewStringMatcher(id int) TokenMatcher

func NewUnixStyleCommentMatcher

func NewUnixStyleCommentMatcher(id int) TokenMatcher

func NewUnknownMatcher

func NewUnknownMatcher(id int) TokenMatcher

func NewWhiteSpaceMatcher

func NewWhiteSpaceMatcher(id int) TokenMatcher

func NewjavaStyleDocMatcher

func NewjavaStyleDocMatcher(id int) TokenMatcher

type TokenPos

type TokenPos struct {
	Line       int
	Col        int
	SourceName string
}

func (TokenPos) String

func (self TokenPos) String() string

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

func NewTokenizer

func NewTokenizer(s string, l *BaseLexer) *Tokenizer

func (*Tokenizer) ConsumeMulti

func (self *Tokenizer) ConsumeMulti(count int)

func (*Tokenizer) ConsumeOne

func (self *Tokenizer) ConsumeOne()

func (*Tokenizer) Count

func (self *Tokenizer) Count() int

func (*Tokenizer) Current

func (self *Tokenizer) Current() rune

func (*Tokenizer) EOF

func (self *Tokenizer) EOF() bool

func (*Tokenizer) Index

func (self *Tokenizer) Index() int

func (*Tokenizer) Line

func (self *Tokenizer) Line() int

func (*Tokenizer) Peek

func (self *Tokenizer) Peek(offset int) rune

func (*Tokenizer) StringRange

func (self *Tokenizer) StringRange(begin, end int) string