lexer

Documentation

Index

• func Tokenize(src string) ([]Token, []LexError)
• type LexError
  • func (e *LexError) Error() string
• type Lexer
• type Token
  • func (t Token) String() string
• type TokenType
  • func (t TokenType) String() string

Functions

func Tokenize

func Tokenize(src string) ([]Token, []LexError)

Tokenize converts DTL source into a token slice.

Types

type LexError

type LexError struct {
	Pos     ast.Position
	Message string
}

LexError records a lexer error with source position.

func (*LexError) Error

func (e *LexError) Error() string

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

Lexer tokenizes DTL source code.

type Token

type Token struct {
	Type TokenType
	Pos  ast.Position
	Val  string // raw text of the token
}

Token is a single lexical token with its source position and value.

func (Token) String

func (t Token) String() string

type TokenType

type TokenType int

TokenType identifies a lexer token.

const (
	// Special
	TokenEOF TokenType = iota
	TokenNewline
	TokenIndent
	TokenDedent

	// Literals
	TokenInt    // 42
	TokenFloat  // 3.14
	TokenString // "hello"
	TokenIdent  // variable_name
	TokenTrue   // true
	TokenFalse  // false
	TokenNull   // null

	// String interpolation
	TokenStringStart // opening " before first interpolation
	TokenStringPart  // string fragment between interpolations
	TokenStringEnd   // closing " after last interpolation
	TokenInterpStart // { inside string
	TokenInterpEnd   // } inside string

	// Keywords
	TokenFn
	TokenLet
	TokenReturn
	TokenIf
	TokenThen
	TokenElse
	TokenMatch
	TokenWhen
	TokenTry
	TokenCatch
	TokenAnd
	TokenOr
	TokenNot
	TokenQuery
	TokenFor
	TokenIn
	TokenRaise
	TokenUse
	TokenRecord
	TokenTypeKw // 'type' keyword for named type definitions

	// Operators
	TokenPlus       // +
	TokenMinus      // -
	TokenStar       // *
	TokenSlash      // /
	TokenPercent    // %
	TokenEq         // ==
	TokenNe         // !=
	TokenGt         // >
	TokenLt         // <
	TokenGe         // >=
	TokenLe         // <=
	TokenAmpAmp     // &&
	TokenPipePipe   // ||
	TokenBang       // !
	TokenPipe       // |
	TokenArrow      // =>
	TokenThinArrow  // ->
	TokenDotDot     // ..
	TokenQQ         // ??
	TokenQDot       // ?.
	TokenPlusPlus   // ++
	TokenDot        // .
	TokenColon      // :
	TokenColonColon // ::
	TokenQuestion   // ?

	// Delimiters
	TokenLParen   // (
	TokenRParen   // )
	TokenLBracket // [
	TokenRBracket // ]
	TokenLBrace   // {
	TokenRBrace   // }
	TokenComma    // ,
	TokenEllipsis // ...
	TokenAssign   // =

)

func (TokenType) String

func (t TokenType) String() string