tokenizer

Documentation

Index

• func Type(t ItemType) rune
• type Item
  • func (i *Item) Errorf(format string, arg ...interface{}) error
  • func (i *Item) HumanName() string
  • func (i *Item) IsExpressionEnd() bool
  • func (i *Item) IsLabel() bool
  • func (i *Item) IsSemiReserved() bool
  • func (i *Item) IsSingle(r rune) bool
  • func (i *Item) Loc() *phpv.Loc
  • func (i *Item) Rune() rune
  • func (i *Item) String() string
  • func (i *Item) Unexpected() error
  • func (i *Item) UnexpectedExpecting(expecting string) error
• type ItemType
  • func Rune(r rune) ItemType
  • func (i ItemType) Name() string
  • func (i ItemType) OpString() string
  • func (i ItemType) Rune() rune
  • func (i ItemType) String() string
• type Lexer
  • func NewLexer(i io.Reader, fn string) *Lexer
  • func NewLexerPhp(i io.Reader, fn string) *Lexer
  • func NewLexerWithShortTag(i io.Reader, fn string, shortOpenTag bool) *Lexer
  • func (l *Lexer) Close()
  • func (l *Lexer) NextItem() (*Item, error)

Functions

func Type

func Type(t ItemType) rune

Types

type Item

type Item struct {
	Type       ItemType
	Data       string
	Filename   string
	Line, Char int
}

func (*Item) Errorf

func (i *Item) Errorf(format string, arg ...interface{}) error

func (*Item) HumanName

func (i *Item) HumanName() string

HumanName returns the PHP 8-style human-readable token name for error messages. For keywords, returns `token "keyword"` (e.g. `token "exit"`). For symbols, returns `"c"` (e.g. `"("`). For special tokens, returns their description (e.g. `end of file`).

func (*Item) IsExpressionEnd

func (i *Item) IsExpressionEnd() bool

func (*Item) IsLabel

func (i *Item) IsLabel() bool

IsLabel returns true if the item represents a PHP label/identifier, including keywords. Used for named arguments where keywords are valid names.

func (*Item) IsSemiReserved

func (i *Item) IsSemiReserved() bool

IsSemiReserved returns true if the token is a semi-reserved keyword that can be used as a class constant name, method name, or property name in PHP. See https://www.php.net/manual/en/reserved.other-reserved-words.php

func (*Item) IsSingle

func (i *Item) IsSingle(r rune) bool

func (*Item) Loc

func (i *Item) Loc() *phpv.Loc

func (*Item) Rune

func (i *Item) Rune() rune

func (*Item) String

func (i *Item) String() string

func (*Item) Unexpected

func (i *Item) Unexpected() error

func (*Item) UnexpectedExpecting

func (i *Item) UnexpectedExpecting(expecting string) error

UnexpectedExpecting returns a parse error with the "expecting" hint.

type ItemType

type ItemType int

const (
	T_EOF                      ItemType
	T_ABSTRACT                 // "abstract"
	T_AND_EQUAL                // "&="
	T_ARRAY                    // array(...
	T_ARRAY_CAST               // "(array)"
	T_AS                       // "as"
	T_BAD_CHARACTER            // anything below ASCII 32 except \t (0x09), \n (0x0a) and \r (0x0d)
	T_BOOLEAN_AND              // "&&"
	T_BOOLEAN_OR               // "||"
	T_BOOL_CAST                // "(bool)" or "(boolean)"
	T_BREAK                    // "break"
	T_CALLABLE                 // "callable"
	T_CASE                     // "case"
	T_CATCH                    // "catch"
	T_CLASS                    // "class"
	T_CLASS_C                  // "__CLASS__"
	T_CLONE                    // "clone"
	T_CLOSE_TAG                // ?> or %>
	T_COALESCE                 // "??"
	T_COALESCE_EQUAL           // "??="
	T_COMMENT                  // // # or /* */
	T_CONCAT_EQUAL             // .=
	T_CONST                    // const
	T_CONSTANT_ENCAPSED_STRING // string in single or double quotes
	T_CONTINUE                 // continue
	T_CURLY_OPEN               // {$ (in double quote strings) see http://php.net/manual/en/language.types.string.php#language.types.string.parsing.complex
	T_DEC                      // "--"
	T_DECLARE                  // "declare"
	T_DEFAULT                  // "default"
	T_DIR                      // "__DIR__"
	T_DIV_EQUAL                // "/="
	T_DOC_COMMENT              // /** */ comments
	T_DO                       // "do"
	T_DOLLAR_OPEN_CURLY_BRACES // ${ see http://php.net/manual/en/language.types.string.php#language.types.string.parsing.complex
	T_DOUBLE_ARROW             // =>
	T_DOUBLE_CAST              // (real), (double), (float)
	T_PAAMAYIM_NEKUDOTAYIM     // "::"
	T_ECHO                     // "echo"
	T_ELLIPSIS                 // ...
	T_ELSE                     // else
	T_ELSEIF
	T_EMPTY                   // empty()
	T_ENCAPSED_AND_WHITESPACE // ?
	T_ENDDECLARE              // enddeclare
	T_ENDFOR                  // endfor
	T_ENDFOREACH              // endforeach
	T_ENDIF                   // endif
	T_ENDSWITCH               // endswitch
	T_ENDWHILE                // endwhile
	T_END_HEREDOC
	T_ENUM // "enum"
	T_EVAL // eval()
	T_EXIT // "exit" or "die"
	T_EXTENDS
	T_FILE // "__FILE__"
	T_FINAL
	T_FINALLY // "finally", for exceptions
	T_FN      // "fn" (arrow functions)
	T_FOR
	T_FOREACH
	T_FUNCTION
	T_FUNC_C // "__FUNCTION__"
	T_GLOBAL // global
	T_GOTO
	T_HALT_COMPILER // __halt_compiler()
	T_IF
	T_IMPLEMENTS
	T_INC
	T_INCLUDE
	T_INCLUDE_ONCE
	T_INLINE_HTML // default type
	T_INSTANCEOF
	T_INSTEADOF
	T_INT_CAST // (int) or (integer)
	T_INTERFACE
	T_ISSET
	T_IS_EQUAL            // ==
	T_IS_GREATER_OR_EQUAL // >=
	T_IS_IDENTICAL        // ===
	T_IS_NOT_EQUAL        // != <>
	T_IS_NOT_IDENTICAL    // !==
	T_IS_SMALLER_OR_EQUAL // <=
	T_SPACESHIP           // <=>
	T_LINE                // "__LINE__"
	T_LIST
	T_LOGICAL_AND  // "and"
	T_LOGICAL_OR   // "or"
	T_LOGICAL_XOR  // "xor"
	T_MATCH        // "match"
	T_METHOD_C     // "__METHOD__"
	T_MINUS_EQUAL  // -=
	T_MOD_EQUAL    // %=
	T_MUL_EQUAL    // *=
	T_NAMESPACE    // "namespace"
	T_NS_C         // __NAMESPACE__
	T_NS_SEPARATOR // \
	T_NEW
	T_NUM_STRING
	T_OBJECT_CAST
	T_NULLSAFE_OBJECT_OPERATOR // ?->
	T_OBJECT_OPERATOR          // ->
	T_OPEN_TAG
	T_OPEN_TAG_WITH_ECHO // "<?="
	T_OR_EQUAL           // |=
	T_PIPE               // |>
	T_PLUS_EQUAL         // +=
	T_POW                // **
	T_POW_EQUAL
	T_PRINT
	T_PRIVATE
	T_PUBLIC
	T_PROTECTED
	T_READONLY // "readonly"
	T_REQUIRE
	T_REQUIRE_ONCE
	T_RETURN
	T_SL // <<
	T_SL_EQUAL
	T_SR // >>
	T_SR_EQUAL
	T_START_HEREDOC
	T_STATIC
	T_STRING      // parent, self, etc. identifiers, e.g. keywords like parent and self, function names, class names and more are matched. See also T_CONSTANT_ENCAPSED_STRING
	T_STRING_CAST // (string)
	T_STRING_VARNAME
	T_SWITCH
	T_THROW
	T_TRAIT
	T_TRAIT_C
	T_TRY
	T_UNSET
	T_UNSET_CAST
	T_USE
	T_VAR
	T_VARIABLE // a variable, eg $foo
	T_WHILE
	T_WHITESPACE
	T_XOR_EQUAL
	T_YIELD
	T_YIELD_FROM
	T_DNUMBER
	T_LNUMBER
	T_ATTRIBUTE  // #[
	T_VOID_CAST  // "(void)"
	T_PROPERTY_C // __PROPERTY__

)

func Rune

func Rune(r rune) ItemType

func (ItemType) Name

func (i ItemType) Name() string

func (ItemType) OpString

func (i ItemType) OpString() string

OpString returns the operator symbol for use in error messages. For compound assignment operators (+=, -= etc.), returns the base operator (+, - etc.) since PHP error messages like "Unsupported operand types" use the base operator.

func (ItemType) Rune

func (i ItemType) Rune() rune

func (ItemType) String

func (i ItemType) String() string

type Lexer

type Lexer struct {
	ShortOpenTag bool // when false, <? without php/= is not a PHP open tag
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(i io.Reader, fn string) *Lexer

func NewLexerPhp

func NewLexerPhp(i io.Reader, fn string) *Lexer

func NewLexerWithShortTag

func NewLexerWithShortTag(i io.Reader, fn string, shortOpenTag bool) *Lexer

func (*Lexer) Close

func (l *Lexer) Close()

Close signals the lexer goroutine to stop. This should be called when compilation is aborted (e.g., on timeout) to prevent goroutine leaks.

func (*Lexer) NextItem

func (l *Lexer) NextItem() (*Item, error)