token

Documentation

Index

• Constants
• func Put(t *Token) error
• type Token
  • func Get() *Token
  • func NewTokenWithModelType(typ Type, literal string) Token
  • func (t Token) HasModelType() bool
  • func (t Token) IsAnyType(types ...models.TokenType) bool
  • func (t Token) IsType(expected models.TokenType) bool
  • func (t *Token) Reset()
• type Type
  • func (t Type) IsKeyword() bool
  • func (t Type) IsLiteral() bool
  • func (t Type) IsOperator() bool
  • func (t Type) ToModelType() models.TokenType

Constants

const (
	// Special tokens
	ILLEGAL = Type("ILLEGAL")
	EOF     = Type("EOF")
	WS      = Type("WS")

	// Identifiers and literals
	IDENT  = Type("IDENT")  // column, table_name
	INT    = Type("INT")    // 12345
	FLOAT  = Type("FLOAT")  // 123.45
	STRING = Type("STRING") // "abc", 'abc'
	TRUE   = Type("TRUE")   // TRUE
	FALSE  = Type("FALSE")  // FALSE

	// Operators
	EQ       = Type("=")
	NEQ      = Type("!=")
	NOT_EQ   = Type("!=") // Alias for NEQ
	LT       = Type("<")
	LTE      = Type("<=")
	GT       = Type(">")
	GTE      = Type(">=")
	ASTERISK = Type("*")

	// Delimiters
	COMMA     = Type(",")
	SEMICOLON = Type(";")
	LPAREN    = Type("(")
	RPAREN    = Type(")")
	DOT       = Type(".")

	// Keywords
	SELECT = Type("SELECT")
	INSERT = Type("INSERT")
	UPDATE = Type("UPDATE")
	DELETE = Type("DELETE")
	FROM   = Type("FROM")
	WHERE  = Type("WHERE")
	ORDER  = Type("ORDER")
	BY     = Type("BY")
	GROUP  = Type("GROUP")
	HAVING = Type("HAVING")
	LIMIT  = Type("LIMIT")
	OFFSET = Type("OFFSET")
	AS     = Type("AS")
	AND    = Type("AND")
	OR     = Type("OR")
	IN     = Type("IN")
	NOT    = Type("NOT")
	NULL   = Type("NULL")
	ALL    = Type("ALL")
	ON     = Type("ON")
	INTO   = Type("INTO")
	VALUES = Type("VALUES")

	// Role keywords
	SUPERUSER    = Type("SUPERUSER")
	NOSUPERUSER  = Type("NOSUPERUSER")
	CREATEDB     = Type("CREATEDB")
	NOCREATEDB   = Type("NOCREATEDB")
	CREATEROLE   = Type("CREATEROLE")
	NOCREATEROLE = Type("NOCREATEROLE")
	LOGIN        = Type("LOGIN")
	NOLOGIN      = Type("NOLOGIN")

	// ALTER statement keywords
	ALTER        = Type("ALTER")
	TABLE        = Type("TABLE")
	ROLE         = Type("ROLE")
	POLICY       = Type("POLICY")
	CONNECTOR    = Type("CONNECTOR")
	ADD          = Type("ADD")
	DROP         = Type("DROP")
	COLUMN       = Type("COLUMN")
	CONSTRAINT   = Type("CONSTRAINT")
	RENAME       = Type("RENAME")
	TO           = Type("TO")
	SET          = Type("SET")
	RESET        = Type("RESET")
	MEMBER       = Type("MEMBER")
	OWNER        = Type("OWNER")
	USER         = Type("USER")
	URL          = Type("URL")
	DCPROPERTIES = Type("DCPROPERTIES")
	CASCADE      = Type("CASCADE")
	WITH         = Type("WITH")
	CHECK        = Type("CHECK")
	USING        = Type("USING")
	UNTIL        = Type("UNTIL")
	VALID        = Type("VALID")
	PASSWORD     = Type("PASSWORD")
	EQUAL        = Type("=")
)

Token types

Functions

func Put

func Put(t *Token) error

Put returns a Token to the pool

Types

type Token

type Token struct {
	Type      Type             // String-based type (backward compatibility)
	ModelType models.TokenType // Int-based type (primary, for performance)
	Literal   string           // The literal value of the token
}

Token represents a lexical token The Token struct supports both string-based (Type) and int-based (ModelType) type systems. ModelType is the primary system going forward, while Type is maintained for backward compatibility.

func Get

func Get() *Token

Get retrieves a Token from the pool

func NewTokenWithModelType

func NewTokenWithModelType(typ Type, literal string) Token

NewTokenWithModelType creates a token with both string and int types populated

func (Token) HasModelType

func (t Token) HasModelType() bool

HasModelType returns true if the ModelType field is populated

func (Token) IsAnyType

func (t Token) IsAnyType(types ...models.TokenType) bool

IsAnyType checks if the token matches any of the given models.TokenType values

func (Token) IsType

func (t Token) IsType(expected models.TokenType) bool

IsType checks if the token matches the given models.TokenType (fast int comparison)

func (*Token) Reset

func (t *Token) Reset()

Reset resets a token's fields

type Type

type Type string

Type represents a token type (string-based, for backward compatibility)

func (Type) IsKeyword

func (t Type) IsKeyword() bool

IsKeyword returns true if the token type is a keyword

func (Type) IsLiteral

func (t Type) IsLiteral() bool

IsLiteral returns true if the token type is a literal

func (Type) IsOperator

func (t Type) IsOperator() bool

IsOperator returns true if the token type is an operator

func (Type) ToModelType

func (t Type) ToModelType() models.TokenType

ToModelType converts a string-based Type to models.TokenType