lexer

Documentation

Index

• func PreprocessUnicodeEscapesContextAware(input string) string
• type Lexer
  • func NewLexer(input string) *Lexer
  • func NewLexerWithSource(sourceFile *source.SourceFile) *Lexer
  • func (l *Lexer) CurrentPosition() int
  • func (l *Lexer) GetSource() *source.SourceFile
  • func (l *Lexer) NextToken() Token
  • func (l *Lexer) RestoreState(state LexerState)
  • func (l *Lexer) SaveState() LexerState
  • func (l *Lexer) SetPosition(pos int)
  • func (l *Lexer) SetRegexContext()
  • func (l *Lexer) SplitGreaterEqualToken(geToken Token) Token
  • func (l *Lexer) SplitRightShiftAssignToken(rsaToken Token) Token
  • func (l *Lexer) SplitRightShiftToken(rsToken Token) Token
  • func (l *Lexer) SplitUnsignedRightShiftToken(ursToken Token) Token
• type LexerState
• type Token
• type TokenType
  • func LookupIdent(ident string) TokenType

Functions

func PreprocessUnicodeEscapesContextAware

func PreprocessUnicodeEscapesContextAware(input string) string

PreprocessUnicodeEscapesContextAware replaces unicode escape sequences with actual characters, but only when NOT inside string literals, comments, or other contexts where they should remain as escapes

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer holds the state of the scanner.

func NewLexer

func NewLexer(input string) *Lexer

NewLexer creates a new Lexer.

func NewLexerWithSource

func NewLexerWithSource(sourceFile *source.SourceFile) *Lexer

func (*Lexer) CurrentPosition

func (l *Lexer) CurrentPosition() int

CurrentPosition returns the lexer's current byte position in the input. Needed for parser backtracking.

func (*Lexer) GetSource

func (l *Lexer) GetSource() *source.SourceFile

GetSource returns the source file associated with this lexer

func (*Lexer) NextToken

func (l *Lexer) NextToken() Token

NextToken scans the input and returns the next token.

func (*Lexer) RestoreState

func (l *Lexer) RestoreState(state LexerState)

RestoreState restores the lexer to a previously saved state

func (*Lexer) SaveState

func (l *Lexer) SaveState() LexerState

SetPosition resets the lexer to a specific byte position and re-reads the character. Needed for parser backtracking. Warning: Does not recalculate line numbers accurately if jumping significantly. Assumes backtracking is local and line changes are minimal or irrelevant for the backtrack. SaveState captures the current lexer state for backtracking

func (*Lexer) SetPosition

func (l *Lexer) SetPosition(pos int)

SetPosition sets lexer position (legacy method, use SaveState/RestoreState for proper backtracking)

func (*Lexer) SetRegexContext

func (l *Lexer) SetRegexContext()

SetRegexContext tells the lexer that the next '/' should be interpreted as a regex literal start. This is called by the parser after parsing statement-ending braces (blocks, functions, classes).

func (*Lexer) SplitGreaterEqualToken

func (l *Lexer) SplitGreaterEqualToken(geToken Token) Token

SplitGreaterEqualToken converts a >= token into > and pushes = back This is used for deeply nested generics followed by assignment like Array<Array<Array<T>>> = []

func (*Lexer) SplitRightShiftAssignToken

func (l *Lexer) SplitRightShiftAssignToken(rsaToken Token) Token

SplitRightShiftAssignToken converts a >>= token into > and pushes >= back This is used for nested generics followed by assignment like Array<Array<T>> = []

func (*Lexer) SplitRightShiftToken

func (l *Lexer) SplitRightShiftToken(rsToken Token) Token

SplitRightShiftToken converts a >> token into > and pushes the second > back This is used for nested generics like Array<Array<T>>

func (*Lexer) SplitUnsignedRightShiftToken

func (l *Lexer) SplitUnsignedRightShiftToken(ursToken Token) Token

SplitUnsignedRightShiftToken converts a >>> token into > and pushes >> back

type LexerState

type LexerState struct {
	Position      int
	ReadPosition  int
	Ch            byte
	Line          int
	Column        int
	InTemplate    bool
	BraceDepth    int
	TemplateStart int
	TemplateStack []templateState
	PushedToken   *Token
	PrevToken     TokenType // Previous token for regex context determination
}

LexerState captures the complete lexer state for backtracking

type Token

type Token struct {
	Type              TokenType
	Literal           string // The actual text of the token (lexeme)
	RawLiteral        string // For template strings: the unprocessed escape sequences (TRV)
	CookedIsUndefined bool   // For template strings: true if cooked value should be undefined (invalid escape)
	Line              int    // 1-based line number where the token starts
	Column            int    // 1-based column number (rune index) where the token starts
	StartPos          int    // 0-based byte offset where the token starts
	EndPos            int    // 0-based byte offset after the token ends
}

Token represents a lexical token.

type TokenType

type TokenType string

TokenType represents the type of a token.

const (
	// Special
	ILLEGAL TokenType = "ILLEGAL" // Unknown token/character
	EOF     TokenType = "EOF"     // End Of File

	// Identifiers + Literals
	IDENT         TokenType = "IDENT"         // functionName, variableName
	PRIVATE_IDENT TokenType = "PRIVATE_IDENT" // #privateName
	NUMBER        TokenType = "NUMBER"        // 123, 45.67
	BIGINT        TokenType = "BIGINT"        // 123n
	STRING        TokenType = "STRING"        // "hello world"
	REGEX_LITERAL TokenType = "REGEX_LITERAL" // /pattern/flags
	NULL          TokenType = "NULL"          // Added
	UNDEFINED     TokenType = "UNDEFINED"     // Added

	// --- NEW: Template Literal Tokens ---
	TEMPLATE_START         TokenType = "TEMPLATE_START"         // ` (opening backtick)
	TEMPLATE_STRING        TokenType = "TEMPLATE_STRING"        // string parts between interpolations
	TEMPLATE_INTERPOLATION TokenType = "TEMPLATE_INTERPOLATION" // ${ (start of interpolation)
	TEMPLATE_END           TokenType = "TEMPLATE_END"           // ` (closing backtick)

	// Operators (add more later)
	ASSIGN   TokenType = "="
	PLUS     TokenType = "+"
	MINUS    TokenType = "-"
	BANG     TokenType = "!"
	ASTERISK TokenType = "*"
	SLASH    TokenType = "/"
	LT       TokenType = "<"
	GT       TokenType = ">"
	EQ       TokenType = "=="
	NOT_EQ   TokenType = "!="
	LE       TokenType = "<="
	GE       TokenType = ">="  // Added (assuming GT might become GE)
	DOT      TokenType = "."   // Added for member access
	SPREAD   TokenType = "..." // Added for spread/rest

	// Compound Assignment
	PLUS_ASSIGN     TokenType = "+=" // Added
	MINUS_ASSIGN    TokenType = "-=" // Added
	ASTERISK_ASSIGN TokenType = "*=" // Added
	SLASH_ASSIGN    TokenType = "/=" // Added

	// --- NEW: Remainder/Exponent Operators ---
	REMAINDER TokenType = "%"
	EXPONENT  TokenType = "**"

	// --- NEW: Remainder/Exponent Assign ---
	REMAINDER_ASSIGN TokenType = "%="
	EXPONENT_ASSIGN  TokenType = "**="

	// Increment/Decrement
	INC TokenType = "++" // Added
	DEC TokenType = "--" // Added

	// --- NEW: Bitwise Operators ---
	BITWISE_AND TokenType = "&"
	// BITWISE_OR           TokenType = "|" // Note: This might conflict with PIPE for Union Types if not handled carefully
	BITWISE_XOR          TokenType = "^"
	BITWISE_NOT          TokenType = "~"
	LEFT_SHIFT           TokenType = "<<"
	RIGHT_SHIFT          TokenType = ">>"
	UNSIGNED_RIGHT_SHIFT TokenType = ">>>"

	// --- NEW: Bitwise Assignment ---
	BITWISE_AND_ASSIGN          TokenType = "&="
	BITWISE_OR_ASSIGN           TokenType = "|="
	BITWISE_XOR_ASSIGN          TokenType = "^="
	LEFT_SHIFT_ASSIGN           TokenType = "<<="
	RIGHT_SHIFT_ASSIGN          TokenType = ">>="
	UNSIGNED_RIGHT_SHIFT_ASSIGN TokenType = ">>>="

	// --- NEW: Logical Assignment ---
	LOGICAL_AND_ASSIGN TokenType = "&&="
	LOGICAL_OR_ASSIGN  TokenType = "||="
	COALESCE_ASSIGN    TokenType = "??="

	// Type Operator
	PIPE TokenType = "|" // Added for Union Types - Retain for clarity, but NextToken needs careful handling

	// Delimiters
	COMMA     TokenType = ","
	SEMICOLON TokenType = ";"
	COLON     TokenType = ":"
	LPAREN    TokenType = "("
	RPAREN    TokenType = ")"
	LBRACE    TokenType = "{"
	RBRACE    TokenType = "}"
	LBRACKET  TokenType = "["  // Added for Arrays
	RBRACKET  TokenType = "]"  // Added for Arrays
	ARROW     TokenType = "=>" // Added for arrow functions
	AT        TokenType = "@"  // Decorator prefix

	// Keywords
	FUNCTION TokenType = "FUNCTION"
	LET      TokenType = "LET"
	CONST    TokenType = "CONST"
	VAR      TokenType = "VAR" // Added
	TRUE     TokenType = "TRUE"
	FALSE    TokenType = "FALSE"
	IF       TokenType = "IF"
	ELSE     TokenType = "ELSE"
	RETURN   TokenType = "RETURN"
	WHILE    TokenType = "WHILE"
	DO       TokenType = "DO" // Added for do...while
	FOR      TokenType = "FOR"
	WITH     TokenType = "WITH"     // Added for with statements
	BREAK    TokenType = "BREAK"    // Added
	CONTINUE TokenType = "CONTINUE" // Added
	TYPE     TokenType = "TYPE"     // Added for type aliases
	SWITCH   TokenType = "SWITCH"   // Added for switch statements
	CASE     TokenType = "CASE"     // Added for switch statements
	DEFAULT  TokenType = "DEFAULT"  // Added for switch statements
	TYPEOF   TokenType = "TYPEOF"   // Added for typeof operator
	VOID     TokenType = "VOID"     // Added for void operator
	KEYOF    TokenType = "KEYOF"    // Added for keyof operator
	INFER    TokenType = "INFER"    // Added for infer keyword in conditional types
	IS       TokenType = "IS"       // Added for type predicates (x is Type)

	// Logical Operators
	LOGICAL_AND TokenType = "&&" // Added
	LOGICAL_OR  TokenType = "||" // Added
	COALESCE    TokenType = "??" // Added

	// New Strict Equality Operators
	STRICT_EQ     TokenType = "==="
	STRICT_NOT_EQ TokenType = "!=="

	// New Ternary Operator Tokens
	QUESTION TokenType = "?"

	// Optional Chaining
	OPTIONAL_CHAINING TokenType = "?."

	// This keyword
	THIS TokenType = "THIS"
	// NEW keyword
	NEW TokenType = "NEW"
	// INTERFACE keyword
	INTERFACE TokenType = "INTERFACE"
	// EXTENDS keyword
	EXTENDS TokenType = "EXTENDS"
	// IMPLEMENTS keyword
	IMPLEMENTS TokenType = "IMPLEMENTS"
	// SUPER keyword
	SUPER TokenType = "SUPER"
	// OF keyword
	OF TokenType = "OF"
	// AS keyword
	AS TokenType = "AS"
	// SATISFIES keyword
	SATISFIES TokenType = "SATISFIES"
	// IN keyword
	IN TokenType = "IN"
	// INSTANCEOF keyword
	INSTANCEOF TokenType = "INSTANCEOF"
	// DELETE keyword
	DELETE TokenType = "DELETE"
	// Exception handling keywords
	TRY      TokenType = "TRY"
	CATCH    TokenType = "CATCH"
	THROW    TokenType = "THROW"
	FINALLY  TokenType = "FINALLY"
	DEBUGGER TokenType = "DEBUGGER"
	// Class keyword
	CLASS TokenType = "CLASS"
	// Enum keyword
	ENUM TokenType = "ENUM"
	// Static keyword (for future use)
	STATIC TokenType = "STATIC"
	// Readonly keyword
	READONLY TokenType = "READONLY"
	// Access modifier keywords
	PUBLIC    TokenType = "PUBLIC"
	PRIVATE   TokenType = "PRIVATE"
	PROTECTED TokenType = "PROTECTED"
	// Abstract and override keywords
	ABSTRACT TokenType = "ABSTRACT"
	OVERRIDE TokenType = "OVERRIDE"
	// Getter/Setter keywords
	GET TokenType = "GET"
	SET TokenType = "SET"
	// Module keywords
	IMPORT TokenType = "IMPORT"
	EXPORT TokenType = "EXPORT"
	FROM   TokenType = "FROM"
	// Generator keyword
	YIELD TokenType = "YIELD"
	// Async/Await keywords
	ASYNC TokenType = "ASYNC"
	AWAIT TokenType = "AWAIT"
)

--- Token Types ---

func LookupIdent

func LookupIdent(ident string) TokenType

LookupIdent checks the keywords table for an identifier.