lexer

Documentation

Overview

Package lexer tokenizes posh shell input.

Index

• type Lexer
  • func New(input string) *Lexer
  • func NewAt(input string, lineBase int) *Lexer
  • func (l *Lexer) Tokenize() []Token
• type Token
• type TokenType
  • func (t TokenType) String() string

Types

type Lexer

type Lexer struct {
	Errors []string // unterminated-string diagnostics
	// contains filtered or unexported fields
}

Lexer holds the tokenizer state.

func New

func New(input string) *Lexer

New creates a Lexer for the given input string.

func NewAt

func NewAt(input string, lineBase int) *Lexer

func (*Lexer) Tokenize

func (l *Lexer) Tokenize() []Token

Tokenize lexes the entire input and returns all tokens including EOF.

type Token

type Token struct {
	Type   TokenType
	Val    string // raw text (for WORD/ASSIGN); empty for punctuation tokens
	Quoted bool   // for HEREDOC_BODY: true = literal body (no expansion)
}

Token is a single lexical unit.

type TokenType

type TokenType int

TokenType identifies the kind of a token.

const (
	WORD              TokenType = iota // bare word, quoted string, or substitution
	ASSIGN                             // VAR=val (only at command start position)
	PIPE                               // |
	AND                                // &&
	OR                                 // ||
	SEMI                               // ;
	AMP                                // &
	REDIR_OUT                          // > or 1>
	REDIR_APPEND                       // >> or 1>>
	REDIR_IN                           // < or 0<
	REDIR_ERR                          // 2>
	REDIR_ERR_APPEND                   // 2>>
	REDIR_BOTH                         // &>
	REDIR_BOTH_APPEND                  // &>>
	REDIR_FD_OUT                       // N>file  (N≠1,2); Val = "N"
	REDIR_FD_APPEND                    // N>>file (N≠1,2); Val = "N"
	REDIR_FD_IN                        // N<file  (N≠0);   Val = "N"
	REDIR_DUP_OUT                      // N>&M;  Val = "N:M"
	REDIR_DUP_IN                       // N<&M;  Val = "N:M"
	REDIR_CLOSE_OUT                    // N>&-;  Val = "N"
	REDIR_CLOSE_IN                     // N<&-;  Val = "N"
	HEREDOC_OP                         // <<
	HEREDOC_STRIP_OP                   // <<-
	HEREDOC_BODY                       // inline heredoc body injected by preprocessHeredocs
	HERESTRING_OP                      // <<<
	LPAREN                             // (
	RPAREN                             // )
	LBRACE                             // {
	RBRACE                             // }
	DLARITH                            // (( arithmetic command ))
	NEWLINE                            // \n
	EOF
)

func (TokenType) String

func (t TokenType) String() string