js_lexer

package
v0.19.9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 10, 2023 License: MIT Imports: 9 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var Keywords = map[string]T{

	"break":      TBreak,
	"case":       TCase,
	"catch":      TCatch,
	"class":      TClass,
	"const":      TConst,
	"continue":   TContinue,
	"debugger":   TDebugger,
	"default":    TDefault,
	"delete":     TDelete,
	"do":         TDo,
	"else":       TElse,
	"enum":       TEnum,
	"export":     TExport,
	"extends":    TExtends,
	"false":      TFalse,
	"finally":    TFinally,
	"for":        TFor,
	"function":   TFunction,
	"if":         TIf,
	"import":     TImport,
	"in":         TIn,
	"instanceof": TInstanceof,
	"new":        TNew,
	"null":       TNull,
	"return":     TReturn,
	"super":      TSuper,
	"switch":     TSwitch,
	"this":       TThis,
	"throw":      TThrow,
	"true":       TTrue,
	"try":        TTry,
	"typeof":     TTypeof,
	"var":        TVar,
	"void":       TVoid,
	"while":      TWhile,
	"with":       TWith,
}
View Source
var StrictModeReservedWords = map[string]bool{
	"implements": true,
	"interface":  true,
	"let":        true,
	"package":    true,
	"private":    true,
	"protected":  true,
	"public":     true,
	"static":     true,
	"yield":      true,
}

Functions

func RangeOfIdentifier

func RangeOfIdentifier(source logger.Source, loc logger.Loc) logger.Range

func RangeOfImportAssertOrWith added in v0.19.3

func RangeOfImportAssertOrWith(source logger.Source, assertOrWith ast.AssertOrWithEntry, which KeyOrValue) logger.Range

Types

type CommentBefore added in v0.18.1

type CommentBefore uint8
const (
	PureCommentBefore CommentBefore = 1 << iota
	KeyCommentBefore
	NoSideEffectsCommentBefore
)

type JSONFlavor added in v0.16.6

type JSONFlavor uint8
const (
	// Specification: https://json.org/
	JSON JSONFlavor = iota

	// TypeScript's JSON superset is not documented but appears to allow:
	// - Comments: https://github.com/microsoft/TypeScript/issues/4987
	// - Trailing commas
	// - Full JS number syntax
	TSConfigJSON

	// This is used by the JavaScript lexer
	NotJSON
)

type KeyOrValue added in v0.19.7

type KeyOrValue uint8
const (
	KeyRange KeyOrValue = iota
	ValueRange
	KeyAndValueRange
)

type Lexer

type Lexer struct {
	LegalCommentsBeforeToken []logger.Range
	CommentsBeforeToken      []logger.Range
	AllComments              []logger.Range
	Identifier               MaybeSubstring

	JSXFactoryPragmaComment      logger.Span
	JSXFragmentPragmaComment     logger.Span
	JSXRuntimePragmaComment      logger.Span
	JSXImportSourcePragmaComment logger.Span
	SourceMappingURL             logger.Span
	BadArrowInTSXSuggestion      string

	Number float64

	ApproximateNewlineCount     int
	CouldBeBadArrowInTSX        int
	BadArrowInTSXRange          logger.Range
	LegacyOctalLoc              logger.Loc
	AwaitKeywordLoc             logger.Loc
	FnOrArrowStartLoc           logger.Loc
	PreviousBackslashQuoteInJSX logger.Range
	LegacyHTMLCommentRange      logger.Range

	Token T

	HasNewlineBefore         bool
	HasCommentBefore         CommentBefore
	IsLegacyOctalLiteral     bool
	PrevTokenWasAwaitKeyword bool

	// The log is disabled during speculative scans that may backtrack
	IsLogDisabled bool
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(log logger.Log, source logger.Source, ts config.TSOptions) Lexer

func NewLexerGlobalName added in v0.8.0

func NewLexerGlobalName(log logger.Log, source logger.Source) Lexer

func NewLexerJSON

func NewLexerJSON(log logger.Log, source logger.Source, json JSONFlavor, errorSuffix string) Lexer

func (*Lexer) AddRangeErrorWithNotes added in v0.14.24

func (lexer *Lexer) AddRangeErrorWithNotes(r logger.Range, text string, notes []logger.MsgData)

func (*Lexer) CookedAndRawTemplateContents added in v0.12.5

func (lexer *Lexer) CookedAndRawTemplateContents() ([]uint16, string)

func (*Lexer) Expect

func (lexer *Lexer) Expect(token T)

func (*Lexer) ExpectContextualKeyword

func (lexer *Lexer) ExpectContextualKeyword(text string)

func (*Lexer) ExpectGreaterThan

func (lexer *Lexer) ExpectGreaterThan(isInsideJSXElement bool)

This parses a single ">" token. If that is the first part of a longer token, this function splits off the first ">" and leaves the remainder of the current token as another, smaller token. For example, ">>=" becomes ">=".

func (*Lexer) ExpectInsideJSXElement

func (lexer *Lexer) ExpectInsideJSXElement(token T)

func (*Lexer) ExpectJSXElementChild

func (lexer *Lexer) ExpectJSXElementChild(token T)

func (*Lexer) ExpectLessThan

func (lexer *Lexer) ExpectLessThan(isInsideJSXElement bool)

This parses a single "<" token. If that is the first part of a longer token, this function splits off the first "<" and leaves the remainder of the current token as another, smaller token. For example, "<<=" becomes "<=".

func (*Lexer) ExpectOrInsertSemicolon

func (lexer *Lexer) ExpectOrInsertSemicolon()

func (*Lexer) Expected

func (lexer *Lexer) Expected(token T)

func (*Lexer) ExpectedString

func (lexer *Lexer) ExpectedString(text string)

func (*Lexer) IsContextualKeyword

func (lexer *Lexer) IsContextualKeyword(text string) bool

func (*Lexer) IsIdentifierOrKeyword

func (lexer *Lexer) IsIdentifierOrKeyword() bool

func (*Lexer) Loc

func (lexer *Lexer) Loc() logger.Loc

func (*Lexer) Next

func (lexer *Lexer) Next()

func (*Lexer) NextInsideJSXElement

func (lexer *Lexer) NextInsideJSXElement()

func (*Lexer) NextJSXElementChild

func (lexer *Lexer) NextJSXElementChild()

func (*Lexer) Range

func (lexer *Lexer) Range() logger.Range

func (*Lexer) Raw

func (lexer *Lexer) Raw() string

func (*Lexer) RescanCloseBraceAsTemplateToken

func (lexer *Lexer) RescanCloseBraceAsTemplateToken()

func (*Lexer) ScanRegExp

func (lexer *Lexer) ScanRegExp()

func (*Lexer) StringLiteral

func (lexer *Lexer) StringLiteral() []uint16

func (*Lexer) SyntaxError

func (lexer *Lexer) SyntaxError()

func (*Lexer) Unexpected

func (lexer *Lexer) Unexpected()

type LexerPanic

type LexerPanic struct{}

type MaybeSubstring added in v0.14.22

type MaybeSubstring struct {
	String string
	Start  ast.Index32
}

This represents a string that is maybe a substring of the current file's "source.Contents" string. The point of doing this is that if it is a substring (the common case), then we can represent it more efficiently.

For compactness and performance, the JS AST represents identifiers as a symbol reference instead of as a string. However, we need to track the string between the first pass and the second pass because the string is only resolved to a symbol in the second pass. To avoid allocating extra memory to store the string, we instead use an index+length slice of the original JS source code. That index is what "Start" represents here. The length is just "len(String)".

Set "Start" to invalid (the zero value) if "String" is not a substring of "source.Contents". This is the case for escaped identifiers. For example, the identifier "fo\u006f" would be "MaybeSubstring{String: "foo"}". It's critical that any code changing the "String" also set "Start" to the zero value, which is best done by just overwriting the whole "MaybeSubstring".

The substring range used to be recovered automatically from the string but that relied on the Go "unsafe" package which can hypothetically break under certain Go compiler optimization passes, so it has been removed and replaced with this more error-prone approach that doesn't use "unsafe".

type T

type T uint8
const (
	TEndOfFile T = iota
	TSyntaxError

	// "#!/usr/bin/env node"
	THashbang

	// Literals
	TNoSubstitutionTemplateLiteral // Contents are in lexer.StringLiteral ([]uint16)
	TNumericLiteral                // Contents are in lexer.Number (float64)
	TStringLiteral                 // Contents are in lexer.StringLiteral ([]uint16)
	TBigIntegerLiteral             // Contents are in lexer.Identifier (string)

	// Pseudo-literals
	TTemplateHead   // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateMiddle // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateTail   // Contents are in lexer.StringLiteral ([]uint16)

	// Punctuation
	TAmpersand
	TAmpersandAmpersand
	TAsterisk
	TAsteriskAsterisk
	TAt
	TBar
	TBarBar
	TCaret
	TCloseBrace
	TCloseBracket
	TCloseParen
	TColon
	TComma
	TDot
	TDotDotDot
	TEqualsEquals
	TEqualsEqualsEquals
	TEqualsGreaterThan
	TExclamation
	TExclamationEquals
	TExclamationEqualsEquals
	TGreaterThan
	TGreaterThanEquals
	TGreaterThanGreaterThan
	TGreaterThanGreaterThanGreaterThan
	TLessThan
	TLessThanEquals
	TLessThanLessThan
	TMinus
	TMinusMinus
	TOpenBrace
	TOpenBracket
	TOpenParen
	TPercent
	TPlus
	TPlusPlus
	TQuestion
	TQuestionDot
	TQuestionQuestion
	TSemicolon
	TSlash
	TTilde

	// Assignments (keep in sync with IsAssign() below)
	TAmpersandAmpersandEquals
	TAmpersandEquals
	TAsteriskAsteriskEquals
	TAsteriskEquals
	TBarBarEquals
	TBarEquals
	TCaretEquals
	TEquals
	TGreaterThanGreaterThanEquals
	TGreaterThanGreaterThanGreaterThanEquals
	TLessThanLessThanEquals
	TMinusEquals
	TPercentEquals
	TPlusEquals
	TQuestionQuestionEquals
	TSlashEquals

	// Class-private fields and methods
	TPrivateIdentifier

	// Identifiers
	TIdentifier     // Contents are in lexer.Identifier (string)
	TEscapedKeyword // A keyword that has been escaped as an identifer

	// Reserved words
	TBreak
	TCase
	TCatch
	TClass
	TConst
	TContinue
	TDebugger
	TDefault
	TDelete
	TDo
	TElse
	TEnum
	TExport
	TExtends
	TFalse
	TFinally
	TFor
	TFunction
	TIf
	TImport
	TIn
	TInstanceof
	TNew
	TNull
	TReturn
	TSuper
	TSwitch
	TThis
	TThrow
	TTrue
	TTry
	TTypeof
	TVar
	TVoid
	TWhile
	TWith
)

If you add a new token, remember to add it to "tokenToString" too

func (T) IsAssign added in v0.8.11

func (t T) IsAssign() bool

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL