lexer

Documentation

Index

• func IsAsciiDigit(ch rune) bool
• func IsDigit(ch rune) bool
• func IsHex(ch rune) bool
• func IsLetter(ch rune) bool
• func IsStringChar(ch rune) bool
• func IsWhitespace(ch rune) bool
• type Lexer
  • func Default() Lexer
• type Token
  • func (tk *Token) String() string
• type TokenKind
• type TokenStream
  • func NewTokenStream(file *diagnostics.File, tokens []*Token) *TokenStream
  • func (ts *TokenStream) Advance()
  • func (ts *TokenStream) AtEnd() bool
  • func (ts *TokenStream) Curr() *Token
  • func (ts *TokenStream) GetFile() *diagnostics.File
  • func (ts *TokenStream) Index() int
  • func (ts *TokenStream) Peek() *Token
  • func (ts *TokenStream) PeekN(n int) *Token
  • func (ts *TokenStream) Reset()
  • func (ts *TokenStream) SeekTo(idx int)

Functions

func IsAsciiDigit

func IsAsciiDigit(ch rune) bool

func IsDigit

func IsDigit(ch rune) bool

func IsHex

func IsHex(ch rune) bool

func IsLetter

func IsLetter(ch rune) bool

func IsStringChar

func IsStringChar(ch rune) bool

func IsWhitespace

func IsWhitespace(ch rune) bool

Types

type Lexer

type Lexer interface {
	diagnostics.MightFail
	Lex(file *diagnostics.File) *TokenStream
}

func Default

func Default() Lexer

type Token

type Token struct {
	Text string
	Kind TokenKind
	Span diagnostics.WordSpan
}

func (*Token) String

func (tk *Token) String() string

type TokenKind

type TokenKind = string

const (
	TokenUnknown TokenKind = "Unknown"

	TokenEOF       TokenKind = "EndOfFile" // sentinel
	TokenIdent     TokenKind = "Identifier"
	TokenInt       TokenKind = "Integer"
	TokenExpoInt   TokenKind = "ExponentInteger"
	TokenFloat     TokenKind = "Float"
	TokenExpoFloat TokenKind = "ExponentFloat"
	TokenHex       TokenKind = "Hex"
	TokenBinary    TokenKind = "Binary"
	TokenString    TokenKind = "String"

	TokenConst  TokenKind = "Const"
	TokenLet    TokenKind = "Let"
	TokenFn     TokenKind = "Fn"
	TokenIf     TokenKind = "If"
	TokenElse   TokenKind = "Else"
	TokenSwitch TokenKind = "Switch"
	TokenCase   TokenKind = "Case"
	TokenRepeat TokenKind = "Repeat"
	TokenWhile  TokenKind = "While"
	TokenFor    TokenKind = "For"
	TokenReturn TokenKind = "Return"
	TokenStruct TokenKind = "Struct"
	TokenEnum   TokenKind = "Enum"
	TokenError  TokenKind = "Error"
	TokenType   TokenKind = "Type"
	TokenExport TokenKind = "Export"
	TokenHidden TokenKind = "Hidden"
	TokenStatic TokenKind = "Static"
	TokenTry    TokenKind = "Try"
	TokenCatch  TokenKind = "Catch"
	TokenMust   TokenKind = "Must"
	TokenTrue   TokenKind = "True"
	TokenFalse  TokenKind = "False"
	TokenInfer  TokenKind = "Infer"
	TokenAs     TokenKind = "As"

	TokenAssign           TokenKind = "Assign"           // =
	TokenPlus             TokenKind = "Plus"             // +
	TokenMinus            TokenKind = "Minus"            // -
	TokenStar             TokenKind = "Star"             // *
	TokenPow              TokenKind = "Power"            // **
	TokenSlash            TokenKind = "Slash"            // /
	TokenPercent          TokenKind = "Percent"          // %
	TokenPlusAssign       TokenKind = "PlusAssign"       // +=
	TokenMinusAssign      TokenKind = "MinusAssign"      // -=
	TokenMultiplyAssign   TokenKind = "MultiplyAssign"   // *=
	TokenDivideAssign     TokenKind = "DivideAssign"     // /=
	TokenModuloAssign     TokenKind = "ModuloAssign"     // %=
	TokenIncrement        TokenKind = "Increment"        // ++
	TokenDecrement        TokenKind = "Decrement"        // --
	TokenEqual            TokenKind = "Equal"            // ==
	TokenNotEqual         TokenKind = "NotEqual"         // !=
	TokenLess             TokenKind = "Less"             // <
	TokenLessEqual        TokenKind = "LessEqual"        // <=
	TokenGreater          TokenKind = "Greater"          // >
	TokenGreaterEqual     TokenKind = "GreaterEqual"     // >=
	TokenNot              TokenKind = "Not"              // !
	TokenLogicalAnd       TokenKind = "LogicalAnd"       // &&
	TokenLogicalOr        TokenKind = "LogicalOr"        // ||
	TokenBitAnd           TokenKind = "BitAnd"           // &
	TokenBitOr            TokenKind = "BitOr"            // |
	TokenBitXor           TokenKind = "BitXor"           // ^
	TokenBitNot           TokenKind = "BitNot"           // ~
	TokenBitAndAssign     TokenKind = "BitAndAssign"     // &=
	TokenBitOrAssign      TokenKind = "BitOrAssign"      // |=
	TokenBitXorAssign     TokenKind = "BitXorAssign"     // ^=
	TokenShiftLeft        TokenKind = "ShiftLeft"        // <<
	TokenShiftRight       TokenKind = "ShiftRight"       // >>
	TokenShiftLeftAssign  TokenKind = "ShiftLeftAssign"  // <<=
	TokenShiftRightAssign TokenKind = "ShiftRightAssign" // >>=
	TokenScope            TokenKind = "Scope"            // ::

	TokenColon      TokenKind = "Colon"      // :
	TokenSemicolon  TokenKind = "SemiColon"  // ;
	TokenUnderscore TokenKind = "Underscore" // _
	TokenPeriod     TokenKind = "Dot"        // .
	TokenRange      TokenKind = "Range"      // ..
	TokenEllipsis   TokenKind = "Ellipsis"   // ...
	TokenQuestion   TokenKind = "Question"   // ?
	TokenLParen     TokenKind = "LParen"     // (
	TokenRParen     TokenKind = "RParen"     // )
	TokenLBrace     TokenKind = "LBrace"     // {
	TokenRBrace     TokenKind = "RBrace"     // }
	TokenLBracket   TokenKind = "LBracket"   // [
	TokenRBracket   TokenKind = "RBracket"   // ]
	TokenComma      TokenKind = "Comma"      // ,
)

type TokenStream

type TokenStream struct {
	// contains filtered or unexported fields
}

func NewTokenStream

func NewTokenStream(file *diagnostics.File, tokens []*Token) *TokenStream

func (*TokenStream) Advance

func (ts *TokenStream) Advance()

func (*TokenStream) AtEnd

func (ts *TokenStream) AtEnd() bool

func (*TokenStream) Curr

func (ts *TokenStream) Curr() *Token

func (*TokenStream) GetFile

func (ts *TokenStream) GetFile() *diagnostics.File

func (*TokenStream) Index

func (ts *TokenStream) Index() int

func (*TokenStream) Peek

func (ts *TokenStream) Peek() *Token

func (*TokenStream) PeekN

func (ts *TokenStream) PeekN(n int) *Token

func (*TokenStream) Reset

func (ts *TokenStream) Reset()

func (*TokenStream) SeekTo

func (ts *TokenStream) SeekTo(idx int)