lexer

Documentation

Index

• type Lexer
  • func New(source string) *Lexer
  • func (l *Lexer) NextToken() Token
  • func (l *Lexer) Parse() []Token
• type Token
• type TokenType

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer stores the current state of the lexer

func New

func New(source string) *Lexer

Creates a new lexer

func (*Lexer) NextToken

func (l *Lexer) NextToken() Token

Creates a token based on the current character and then advances the lexer

func (*Lexer) Parse

func (l *Lexer) Parse() []Token

Parses the entire source code, returning the tokens it generated

type Token

type Token struct {
	Type    TokenType
	Literal string
	Line    int
}

Represents a token in the lexer

type TokenType

type TokenType string

const (
	TokenIdent TokenType = "IDENTIFIER"

	TokenIntLiteral   TokenType = "INTEGER"
	TokenFloatLiteral TokenType = "FLOAT"
	TokenCharLiteral  TokenType = "CHAR"

	TokenLParen TokenType = "("
	TokenRParen TokenType = ")"
	TokenLBrace TokenType = "{"
	TokenRBrace TokenType = "}"

	TokenSemicolon TokenType = ";"
	TokenComma     TokenType = ","

	TokenEqual       TokenType = "="
	TokenDash        TokenType = "-"
	TokenPlus        TokenType = "+"
	TokenStar        TokenType = "*"
	TokenSlash       TokenType = "/"
	TokenLessThan    TokenType = "<"
	TokenGreaterThan TokenType = ">"
	TokenBang        TokenType = "!"

	TokenPlusEqual      TokenType = "+="
	TokenMinusEqual     TokenType = "-="
	TokenTimesEqual     TokenType = "*="
	TokenDivideEqual    TokenType = "/="
	TokenNotEqual       TokenType = "!="
	TokenLessOrEqual    TokenType = "<="
	TokenGreaterOrEqual TokenType = ">="
	TokenEqualEqual     TokenType = "=="
	TokenIncrement      TokenType = "++"
	TokenDecrement      TokenType = "--"

	TokenIntKeyword   TokenType = "int"
	TokenFloatKeyword TokenType = "float"
	TokenCharKeyword  TokenType = "char"
	TokenReturn       TokenType = "return"
	TokenConst        TokenType = "const"
	TokenExtern       TokenType = "extern"
	TokenVoid         TokenType = "void"
	TokenIf           TokenType = "if"
	TokenElse         TokenType = "else"
	TokenFor          TokenType = "for"
	TokenWhile        TokenType = "while"
	TokenDo           TokenType = "do"

	TokenEndOfFile TokenType = "EOF"
	TokenIllegal   TokenType = "ILLEGAL"
)

All tokens the lexer can read