tokenize

package
v0.0.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 8, 2022 License: MIT Imports: 6 Imported by: 0

Documentation

Index

Constants

View Source 
const FixedDecimals = 3

TODO: Hack. It is a bit of a hack to set the number of decimals in precision here

View Source 
const SpacesForIndentation = 4

Variables

This section is empty.

Functions

func AddInvisibleCharacters 

func AddInvisibleCharacters(s string) string

func DetectLowercaseKeyword 

func DetectLowercaseKeyword(t token.VariableSymbolToken) (token.Keyword, error)

func DetectUppercaseBoolean 

func DetectUppercaseBoolean(t token.TypeSymbolToken) (token.BooleanToken, error)

func IsDedent 

func IsDedent(report token.IndentationReport, enforceStyle bool) bool

func LegalContinuationSpace 

func LegalContinuationSpace(report token.IndentationReport, enforceStyle bool) bool

func LegalContinuationSpaceIndentation 

func LegalContinuationSpaceIndentation(report token.IndentationReport, indentation int, enforceStyle bool) bool

func LegalSameIndentationOrNoSpace 

func LegalSameIndentationOrNoSpace(report token.IndentationReport, indentation int, enforceStyle bool) bool

func LegalSameIndentationOrOptionalOneSpace 

func LegalSameIndentationOrOptionalOneSpace(report token.IndentationReport, indentation int, enforceStyle bool) bool

func NewTokenizer 

func NewTokenizer(r *runestream.RuneReader, exactWhitespace bool) (*Tokenizer, TokenError)

NewTokenizer :

func NewTokenizerInternal 

func NewTokenizerInternal(r *runestream.RuneReader, exactWhitespace bool) (*Tokenizer, TokenError)

NewTokenizerInternal :

func NewTokenizerInternalWithStartPosition added in v0.0.4 

func NewTokenizerInternalWithStartPosition(r *runestream.RuneReader, position token.Position, exactWhitespace bool) (*Tokenizer, TokenError)

NewTokenizerInternalWithStartPosition :

Types

type CommentAllowedType 

type CommentAllowedType int
const (
	SameLine CommentAllowedType = iota
	OwnLine
	NotAllowedAtAll
)

type CommentNotAllowedHereError 

type CommentNotAllowedHereError struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewCommentNotAllowedHereError 

func NewCommentNotAllowedHereError(posLength token.SourceFileReference, subError error) CommentNotAllowedHereError

func (CommentNotAllowedHereError) Error 

func (e CommentNotAllowedHereError) Error() string

type EncounteredEOF 

type EncounteredEOF struct {
	StandardTokenError
}

func NewEncounteredEOF 

func NewEncounteredEOF() EncounteredEOF

func (EncounteredEOF) Error 

func (e EncounteredEOF) Error() string

type EndOfFile 

type EndOfFile struct {
	token.SourceFileReference
}

func (*EndOfFile) FetchPositionLength 

func (e *EndOfFile) FetchPositionLength() token.SourceFileReference

func (*EndOfFile) Position 

func (e *EndOfFile) Position() token.Position

func (*EndOfFile) String 

func (e *EndOfFile) String() string

func (*EndOfFile) Type 

func (e *EndOfFile) Type() token.Type

type ExpectedIndentationAfterNewLineError 

type ExpectedIndentationAfterNewLineError struct {
	SubError
	// contains filtered or unexported fields
}

func NewExpectedIndentationAfterNewLineError 

func NewExpectedIndentationAfterNewLineError(expectedIndentation int, encounteredIndentation int, err TokenError) ExpectedIndentationAfterNewLineError

func (ExpectedIndentationAfterNewLineError) Error 

func (e ExpectedIndentationAfterNewLineError) Error() string

type ExpectedIndentationError 

type ExpectedIndentationError struct {
	SubError
}

func NewExpectedIndentationError 

func NewExpectedIndentationError(err TokenError) ExpectedIndentationError

func (ExpectedIndentationError) Error 

func (e ExpectedIndentationError) Error() string

type ExpectedNewLineAndIndentationError 

type ExpectedNewLineAndIndentationError struct {
	SubError
	// contains filtered or unexported fields
}

func NewExpectedNewLineAndIndentationError 

func NewExpectedNewLineAndIndentationError(expectedIndentation int, encounteredIndentation int, err TokenError) ExpectedNewLineAndIndentationError

func (ExpectedNewLineAndIndentationError) Error 

func (e ExpectedNewLineAndIndentationError) Error() string

type ExpectedNewLineError 

type ExpectedNewLineError struct {
	// contains filtered or unexported fields
}

func NewExpectedNewLineError 

func NewExpectedNewLineError(eatError TokenError) ExpectedNewLineError

func (ExpectedNewLineError) Error 

func (e ExpectedNewLineError) Error() string

func (ExpectedNewLineError) FetchPositionLength 

func (e ExpectedNewLineError) FetchPositionLength() token.SourceFileReference

type ExpectedOneSpaceError 

type ExpectedOneSpaceError struct {
	SubError
}

func NewExpectedOneSpaceError 

func NewExpectedOneSpaceError(err TokenError) ExpectedOneSpaceError

func (ExpectedOneSpaceError) Error 

func (e ExpectedOneSpaceError) Error() string

type ExpectedTypeSymbolError 

type ExpectedTypeSymbolError struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewExpectedTypeSymbolError 

func NewExpectedTypeSymbolError(posLength token.SourceFileReference, encountered string) ExpectedTypeSymbolError

func (ExpectedTypeSymbolError) Error 

func (e ExpectedTypeSymbolError) Error() string

type ExpectedVariableSymbolError 

type ExpectedVariableSymbolError struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewExpectedVariableSymbolError 

func NewExpectedVariableSymbolError(posLength token.SourceFileReference, encountered string) ExpectedVariableSymbolError

func (ExpectedVariableSymbolError) Error 

func (e ExpectedVariableSymbolError) Error() string

type IllegalCharacterError 

type IllegalCharacterError struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewIllegalCharacterError 

func NewIllegalCharacterError(posLength token.SourceFileReference, encountered rune) IllegalCharacterError

func (IllegalCharacterError) Error 

func (e IllegalCharacterError) Error() string

type IllegalIndentationError 

type IllegalIndentationError struct {
	// contains filtered or unexported fields
}

func NewIllegalIndentationError 

func NewIllegalIndentationError(posLength token.SourceFileReference, encounteredSpaces int, multiples int) IllegalIndentationError

func (IllegalIndentationError) Error 

func (e IllegalIndentationError) Error() string

func (IllegalIndentationError) FetchPositionLength 

func (e IllegalIndentationError) FetchPositionLength() token.SourceFileReference

type InternalError 

type InternalError struct {
	// contains filtered or unexported fields
}

func NewInternalError 

func NewInternalError(err error) InternalError

func (InternalError) Error 

func (e InternalError) Error() string

func (InternalError) FetchPositionLength 

func (e InternalError) FetchPositionLength() token.SourceFileReference

type LineIsLongerThanRecommendedError added in v0.0.4 

type LineIsLongerThanRecommendedError struct {
	StandardTokenError
}

func NewLineIsLongerThanRecommendedError added in v0.0.4 

func NewLineIsLongerThanRecommendedError(posLength token.SourceFileReference) LineIsLongerThanRecommendedError

func (LineIsLongerThanRecommendedError) Error added in v0.0.4 

func (e LineIsLongerThanRecommendedError) Error() string

type LineIsTooLongError added in v0.0.4 

type LineIsTooLongError struct {
	StandardTokenError
}

func NewLineIsTooLongError added in v0.0.4 

func NewLineIsTooLongError(posLength token.SourceFileReference) LineIsTooLongError

func (LineIsTooLongError) Error added in v0.0.4 

func (e LineIsTooLongError) Error() string

type MultiErrors added in v0.0.4 

type MultiErrors struct {
	// contains filtered or unexported fields
}

func NewMultiErrors added in v0.0.4 

func NewMultiErrors(errors []TokenError) *MultiErrors

func (MultiErrors) Error added in v0.0.4 

func (e MultiErrors) Error() string

func (MultiErrors) Errors added in v0.0.4 

func (e MultiErrors) Errors() []TokenError

func (MultiErrors) FetchPositionLength added in v0.0.4 

func (e MultiErrors) FetchPositionLength() token.SourceFileReference

type NotAParenToken added in v0.0.2 

type NotAParenToken struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewNotAParenToken added in v0.0.2 

func NewNotAParenToken(posLength token.SourceFileReference, encounteredToken token.Token) NotAParenToken

func (NotAParenToken) Error added in v0.0.2 

func (e NotAParenToken) Error() string

type NotAnOpenOperatorError added in v0.0.2 

type NotAnOpenOperatorError struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewNotAnOpenOperatorError added in v0.0.2 

func NewNotAnOpenOperatorError(posLength token.SourceFileReference, encounteredRune rune) NotAnOpenOperatorError

func (NotAnOpenOperatorError) Error added in v0.0.2 

func (e NotAnOpenOperatorError) Error() string

type NotEndToken added in v0.0.2 

type NotEndToken struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewNotEndToken added in v0.0.2 

func NewNotEndToken(posLength token.SourceFileReference, encounteredToken rune) NotEndToken

func (NotEndToken) Error added in v0.0.2 

func (e NotEndToken) Error() string

type ParseNumberType added in v0.0.4 

type ParseNumberType uint8
const (
	IntType ParseNumberType = iota
	FixedType
	HexType
	BinaryType
)

type StandardTokenError 

type StandardTokenError struct {
	// contains filtered or unexported fields
}

func (StandardTokenError) FetchPositionLength 

func (e StandardTokenError) FetchPositionLength() token.SourceFileReference

type SubError 

type SubError struct {
	SubErr TokenError
}

func NewSubError 

func NewSubError(err TokenError) SubError

func (SubError) FetchPositionLength 

func (e SubError) FetchPositionLength() token.SourceFileReference

type TokenError 

type TokenError interface {
	FetchPositionLength() token.SourceFileReference
	Error() string
}

func LegalOneSpaceOrNewLineIndentation added in v0.0.2 

func LegalOneSpaceOrNewLineIndentation(report token.IndentationReport, indentation int, enforceStyle bool) (bool, TokenError)

func LegalOneSpaceOrSameIndentation added in v0.0.2 

func LegalOneSpaceOrSameIndentation(report token.IndentationReport, indentation int, enforceStyle bool) (bool, TokenError)

type Tokenizer 

type Tokenizer struct {
	// contains filtered or unexported fields
}

Tokenizer :

func (*Tokenizer) DebugInfo 

func (t *Tokenizer) DebugInfo() string

func (*Tokenizer) DebugInfoLinesWithComment 

func (t *Tokenizer) DebugInfoLinesWithComment(s string, rowCount int) string

func (*Tokenizer) DebugInfoWithComment 

func (t *Tokenizer) DebugInfoWithComment(s string) string

func (*Tokenizer) DebugPrint 

func (t *Tokenizer) DebugPrint(s string)

func (*Tokenizer) DetectRune 

func (t *Tokenizer) DetectRune(requiredRune rune) bool

func (*Tokenizer) DetectString 

func (t *Tokenizer) DetectString(s string) bool

func (*Tokenizer) Document 

func (t *Tokenizer) Document() *token.SourceFileDocument

func (*Tokenizer) EatOneSpace 

func (t *Tokenizer) EatOneSpace() (token.IndentationReport, TokenError)

func (*Tokenizer) EatRune 

func (t *Tokenizer) EatRune(requiredRune rune) TokenError

func (*Tokenizer) EatString 

func (t *Tokenizer) EatString(requiredString string) TokenError

func (*Tokenizer) ExtractStrings 

func (t *Tokenizer) ExtractStrings(startRow int, rowCount int) []string

func (*Tokenizer) MakeRangeMinusOne added in v0.0.2 

func (t *Tokenizer) MakeRangeMinusOne(pos token.PositionToken) token.Range

func (*Tokenizer) MakeSourceFileReference 

func (t *Tokenizer) MakeSourceFileReference(pos token.PositionToken) token.SourceFileReference

func (*Tokenizer) MaybeAccessor 

func (t *Tokenizer) MaybeAccessor() bool

func (*Tokenizer) MaybeAssign 

func (t *Tokenizer) MaybeAssign() bool

func (*Tokenizer) MaybeEOF 

func (t *Tokenizer) MaybeEOF() bool

func (*Tokenizer) MaybeOneNewLine 

func (t *Tokenizer) MaybeOneNewLine() bool

func (*Tokenizer) MaybeRune 

func (t *Tokenizer) MaybeRune(requiredRune rune) bool

func (*Tokenizer) MaybeSpacingRune 

func (t *Tokenizer) MaybeSpacingRune(requiredRune rune) bool

func (*Tokenizer) MaybeString 

func (t *Tokenizer) MaybeString(requiredString string) bool

func (*Tokenizer) ParseCharacter 

func (t *Tokenizer) ParseCharacter(startPosition token.PositionToken) (token.CharacterToken, TokenError)

func (*Tokenizer) ParseNumber 

func (t *Tokenizer) ParseNumber(a string) (token.NumberToken, TokenError)

func (*Tokenizer) ParseOperator 

func (t *Tokenizer) ParseOperator() (token.Token, TokenError)

func (*Tokenizer) ParseSpecialKeyword 

func (t *Tokenizer) ParseSpecialKeyword(pos token.PositionToken) (token.Token, TokenError)

func (*Tokenizer) ParseStartingKeyword 

func (t *Tokenizer) ParseStartingKeyword() (token.Token, TokenError)

func (*Tokenizer) ParseString 

func (t *Tokenizer) ParseString(startStringRune rune, startPosition token.PositionToken) (token.StringToken, TokenError)

func (*Tokenizer) ParseStringInterpolationString added in v0.0.4 

func (t *Tokenizer) ParseStringInterpolationString(stringToken token.StringToken) (token.StringInterpolationString, TokenError)

func (*Tokenizer) ParseStringInterpolationTuple added in v0.0.4 

func (t *Tokenizer) ParseStringInterpolationTuple(stringToken token.StringToken) (token.StringInterpolationTuple, TokenError)

func (*Tokenizer) ParseTypeSymbol 

func (t *Tokenizer) ParseTypeSymbol() (token.TypeSymbolToken, TokenError)

func (*Tokenizer) ParseUnaryOperator 

func (t *Tokenizer) ParseUnaryOperator() (token.Token, TokenError)

func (*Tokenizer) ParseVariableSymbol 

func (t *Tokenizer) ParseVariableSymbol() (token.VariableSymbolToken, TokenError)

func (*Tokenizer) ParsingPosition 

func (t *Tokenizer) ParsingPosition() token.PositionToken

func (*Tokenizer) ReadAnyOperator 

func (t *Tokenizer) ReadAnyOperator() (token.Token, TokenError)

func (*Tokenizer) ReadEndOrSeparatorToken 

func (t *Tokenizer) ReadEndOrSeparatorToken() (token.Token, TokenError)

func (*Tokenizer) ReadMultilineComment 

func (t *Tokenizer) ReadMultilineComment(positionToken token.PositionToken) (token.MultiLineCommentToken, TokenError)

func (*Tokenizer) ReadOpenOperatorToken 

func (t *Tokenizer) ReadOpenOperatorToken(r rune, singleCharLength token.SourceFileReference) (token.Token, TokenError)

func (*Tokenizer) ReadSingleLineComment 

func (t *Tokenizer) ReadSingleLineComment(positionToken token.PositionToken) token.MultiLineCommentToken

func (*Tokenizer) ReadStringUntilEndOfLine 

func (t *Tokenizer) ReadStringUntilEndOfLine() string

func (*Tokenizer) ReadStringUntilEndOfMultilineComment 

func (t *Tokenizer) ReadStringUntilEndOfMultilineComment(pos token.PositionToken) (token.MultiLineCommentToken, TokenError)

func (*Tokenizer) ReadTermToken 

func (t *Tokenizer) ReadTermToken() (token.Token, TokenError)

func (*Tokenizer) ReadTermTokenOrEndOrSeparator 

func (t *Tokenizer) ReadTermTokenOrEndOrSeparator() (token.Token, error)

func (*Tokenizer) RelativeFilename 

func (t *Tokenizer) RelativeFilename() string

func (*Tokenizer) Seek 

func (t *Tokenizer) Seek(pos int)

func (*Tokenizer) SkipWhitespaceAllowCommentsToNextIndentation 

func (t *Tokenizer) SkipWhitespaceAllowCommentsToNextIndentation() (token.IndentationReport, TokenError)

func (*Tokenizer) SkipWhitespaceToNextIndentation 

func (t *Tokenizer) SkipWhitespaceToNextIndentation() (token.IndentationReport, TokenError)

func (*Tokenizer) SkipWhitespaceToNextIndentationHelper 

func (t *Tokenizer) SkipWhitespaceToNextIndentationHelper(allowComments CommentAllowedType) (token.IndentationReport, TokenError)

func (*Tokenizer) SourceFile 

func (t *Tokenizer) SourceFile() *token.SourceFileURI

func (*Tokenizer) Tell 

func (t *Tokenizer) Tell() int

func (*Tokenizer) WasDefaultSymbol 

func (t *Tokenizer) WasDefaultSymbol() (token.RuneToken, bool)

func (*Tokenizer) WasRune 

func (t *Tokenizer) WasRune(requiredRune rune) (token.RuneToken, bool)

func (*Tokenizer) WasSpacingRune 

func (t *Tokenizer) WasSpacingRune(requiredRune rune) (token.RuneToken, bool)

type TokenizerError 

type TokenizerError struct {
	// contains filtered or unexported fields
}

func (TokenizerError) Error 

func (f TokenizerError) Error() string

func (TokenizerError) FetchPositionLength 

func (f TokenizerError) FetchPositionLength() token.SourceFileReference

type TrailingSpaceError 

type TrailingSpaceError struct {
	StandardTokenError
}

func NewTrailingSpaceError 

func NewTrailingSpaceError(posLength token.SourceFileReference) TrailingSpaceError

func (TrailingSpaceError) Error 

func (e TrailingSpaceError) Error() string

type UnexpectedEatTokenError 

type UnexpectedEatTokenError struct {
	StandardTokenError
	// contains filtered or unexported fields
}

func NewUnexpectedEatTokenError 

func NewUnexpectedEatTokenError(posLength token.SourceFileReference, requiredRune rune, encounteredRune rune) UnexpectedEatTokenError

func (UnexpectedEatTokenError) Error 

func (e UnexpectedEatTokenError) Error() string

type UnexpectedIndentationError 

type UnexpectedIndentationError struct {
	// contains filtered or unexported fields
}

func NewUnexpectedIndentationError 

func NewUnexpectedIndentationError(posLength token.SourceFileReference, requiredIndentation int, encounteredIndentation int) UnexpectedIndentationError

func (UnexpectedIndentationError) Error 

func (e UnexpectedIndentationError) Error() string

func (UnexpectedIndentationError) FetchPositionLength 

func (e UnexpectedIndentationError) FetchPositionLength() token.SourceFileReference

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.