lexer

Documentation

Index

• func PosJSON(p Position) map[string]interface{}
• type Error
  • func (e Error) Error() string
• type Lexer
  • func New(source string) *Lexer
  • func (l *Lexer) Lex() []Token
• type Position
• type SimplePosition
• type SimpleToken
  • func ToSimpleTokens(tokens []Token) []SimpleToken
• type Token
  • func (t Token) FullEnd() int
  • func (t Token) FullStart() int
  • func (t Token) IsTrivia() bool
  • func (t Token) ToJSON() map[string]interface{}
• type TokenKind

Functions

func PosJSON

func PosJSON(p Position) map[string]interface{}

Types

type Error

type Error struct {
	Message  string   `json:"message"`
	Position Position `json:"position"`
}

func (Error) Error

func (e Error) Error() string

type Lexer

type Lexer struct {
	Errors []Error
	// contains filtered or unexported fields
}

func New

func New(source string) *Lexer

func (*Lexer) Lex

func (l *Lexer) Lex() []Token

Lex returns significant tokens with trivia attached.

type Position

type Position struct {
	Offset int `json:"offset"` // rune offset from start of source
	Line   int `json:"line"`
	Column int `json:"column"` // rune offset from start of line
}

type SimplePosition

type SimplePosition struct {
	Line   int `json:"line"`
	Column int `json:"column"`
}

type SimpleToken

type SimpleToken struct {
	Kind     TokenKind      `json:"kind"`
	Value    string         `json:"value"`
	Position SimplePosition `json:"position"`
}

func ToSimpleTokens

func ToSimpleTokens(tokens []Token) []SimpleToken

type Token

type Token struct {
	Kind            TokenKind `json:"kind"`
	Value           string    `json:"value"`
	StartPos        Position  `json:"startPos"`
	EndPos          Position  `json:"endPos"`
	LeadingTrivia   []Token   `json:"leadingTrivia"`
	TrailingTrivia  []Token   `json:"trailingTrivia"`
	LeadingInvalid  []Token   `json:"leadingInvalid"`
	TrailingInvalid []Token   `json:"trailingInvalid"`
	IsInvalid       bool      `json:"isInvalid"`
	Start           int       `json:"start"` // rune offset
	End             int       `json:"end"`   // rune offset
}

Token is a lexical token with trivia attached.

func (Token) FullEnd

func (t Token) FullEnd() int

func (Token) FullStart

func (t Token) FullStart() int

func (Token) IsTrivia

func (t Token) IsTrivia() bool

func (Token) ToJSON

func (t Token) ToJSON() map[string]interface{}

ToJSON returns the token in the parser-JSON format (with trivia).

type TokenKind

type TokenKind string

const (
	KindIdentifier   TokenKind = "<identifier>"
	KindString       TokenKind = "<string>"
	KindNumeric      TokenKind = "<numeric-literal>"
	KindColor        TokenKind = "<color-literal>"
	KindFuncExpr     TokenKind = "<function-expression>"
	KindQuotedString TokenKind = "<quoted-string>"
	KindLBrace       TokenKind = "<lbrace>"
	KindRBrace       TokenKind = "<rbrace>"
	KindLBracket     TokenKind = "<lbracket>"
	KindRBracket     TokenKind = "<rbracket>"
	KindLParen       TokenKind = "<lparen>"
	KindRParen       TokenKind = "<rparen>"
	KindColon        TokenKind = "<colon>"
	KindComma        TokenKind = "<comma>"
	KindSemicolon    TokenKind = "<semicolon>"
	KindOp           TokenKind = "<op>"
	KindEOF          TokenKind = "<eof>"

	// trivia
	KindSpace         TokenKind = "<space>"
	KindTab           TokenKind = "<tab>"
	KindNewline       TokenKind = "<newline>"
	KindSingleComment TokenKind = "<single-line-comment>"
	KindMultiComment  TokenKind = "<multiline-comment>"
)