scanner

package module
v0.0.0-...-f8b1171 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 17, 2023 License: MIT Imports: 13 Imported by: 0

README

scanner

Work in Progress

Documentation

Index

Examples

Constants

View Source 
const (
	BinKind     = "bin"
	CommentKind = "comment"
	ErrKind     = "err"
	RealKind    = "real"
	ExpKind     = "exp"
	HexKind     = "hex"
	IdentKind   = "ident"
	IllegalKind = "illegal"
	IntKind     = "int"
	NumKind     = "num"
	OctKind     = "oct"
	StrKind     = "str"
)

General token kinds

Variables

View Source 
var (
	MotorolaBin = Prefix("%", Bin)
	MotorolaOct = Prefix("@", Oct)
	MotorolaHex = Prefix("$", Hex)

	GoBin = Rules(
		Prefix("0b", Bin.WithDigitSep(Rune('_'))),
		Prefix("0B", Bin.WithDigitSep(Rune('_'))),
	)
	GoOct = Rules(
		Prefix("0o", Oct.WithDigitSep(Rune('_'))),
		Prefix("0O", Oct.WithDigitSep(Rune('_'))),
	)
	GoHex = Rules(
		Prefix("0x", goHex),
		Prefix("0X", goHex),
	)
)
View Source 
var (
	StrDoubleQuote = Str('"', '"')
	StrSingleQuote = Str('\'', '\'')
)

Functions

func FormatTokenTable 

func FormatTokenTable(ts []Token) string

func Main 

func Main(fn func(io.Reader) []Token)

func MainWithArgs 

func MainWithArgs(fn func(io.Reader) []Token, args []string)

func Repeat 

func Repeat(fn func(), n int)

Repeat calls function fn in a loop for n times.

func RunTests 

func RunTests(t *testing.T, rules RuleSet, tests []TestCase)

func While 

func While(s *Scanner, c Class, fn func())

Types

type Class 

type Class func(rune) bool

Class returns true if the given rune is a member. 

var (
	// IsAny returns true as long as there is a rune available in the input
	// stream.
	IsAny Class = func(r rune) bool {
		return r != end
	}

	// IsCurrency returns true when given a rune that is a currency symbol as
	// defined by Unicode.
	IsCurrency Class = func(r rune) bool {
		return unicode.Is(unicode.Sc, r)
	}

	// IsDigit returns true when given a digit as defined by Unicode.
	IsDigit Class = unicode.IsDigit

	// IsDigit01 returns true when given a valid binary digit.
	IsDigit01 Class = Rune('0', '1')

	// IsDigit07 returns true when given a valid octal digit.
	IsDigit07 Class = Range('0', '7')

	// IsDigit09 returns true when given a valid decimal digit.
	IsDigit09 Class = Range('0', '9')

	// IsDigit0F returns true when given a valid hexadecimal digit.
	IsDigit0F Class = Or(
		IsDigit09,
		Range('a', 'f'),
		Range('A', 'F'),
	)

	// IsLetter returns true when given rune is a letter as defined by Unicode.
	IsLetter Class = unicode.IsLetter

	// IsLetterAZ returns true when given letters from the Latin alphabet.
	IsLetterAZ Class = Or(
		Range('a', 'z'),
		Range('A', 'Z'),
	)

	// IsLetterUnder returns true when given letters as defined by Unicode
	// or an underscore.
	IsLetterUnder = Or(
		IsLetter,
		Rune('_'),
	)

	// IsLetterDigitUnder returns true when given letters as digits as defined
	// by Unicode or an underscore.
	IsLetterDigitUnder = Or(
		IsLetterUnder,
		IsDigit,
	)

	// IsNone always returns false.
	IsNone Class = func(r rune) bool {
		return false
	}

	// IsPrintable returns true when given a rune that is printable as defined
	// by Unicode.
	IsPrintable Class = unicode.IsPrint

	// IsRune8 returns true when given a rune that can be represented by
	// an 8-bit number.
	IsRune8 Class = Range(0, 0xff)

	// IsRune16 returns true when given a rune that can be represented by
	// a 16-bit number.
	IsRune16 Class = Range(0, 0xffff)

	IsSign Class = Rune('+', '-')

	// IsWhitespace returns true when the given rune is whitespace as
	// defined by Unicode.
	IsWhitespace Class = unicode.IsSpace
)

func Not 

func Not(c Class) Class

Not returns a Class function that returns true when given a rune that does not match class c.

Example 
isA := Rune('a')
isNotA := Not(isA)
fmt.Printf("%c: %v\n", 'a', isNotA('a'))
fmt.Printf("%c: %v\n", 'b', isNotA('b'))

Output:

a: false
b: true

func Or 

func Or(cs ...Class) Class

Or returns a Class that returns true when given a rune that matches any class found in cs.

Example 
isLowerAZ := Range('a', 'z')
isUpperAZ := Range('A', 'Z')
isLetterAZ := Or(isLowerAZ, isUpperAZ)
fmt.Printf("%c: %v\n", 'f', isLetterAZ('f'))
fmt.Printf("%c: %v\n", 'F', isLetterAZ('F'))
fmt.Printf("%c: %v\n", '4', isLetterAZ('4'))

Output:

f: true
F: true
4: false

func Range 

func Range(from rune, to rune) Class

Range returns a Class function that returns true when given a rune that is between from and to inclusive.

Example 
isDigit09 := Range('0', '9')
fmt.Printf("%c: %v\n", '3', isDigit09('3'))
fmt.Printf("%c: %v\n", '6', isDigit09('6'))
fmt.Printf("%c: %v\n", 'a', isDigit09('a'))

Output:

3: true
6: true
a: false

func Rune 

func Rune(rs ...rune) Class

Rune returns a Class function that returns true when given any rune found in rs.

Example 
isAB := Rune('a', 'b')
fmt.Printf("%c: %v\n", 'a', isAB('a'))
fmt.Printf("%c: %v\n", 'b', isAB('b'))
fmt.Printf("%c: %v\n", 'c', isAB('c'))

Output:

a: true
b: true
c: false

type CommentRule 

type CommentRule struct {
	// contains filtered or unexported fields
}
var (
	CppLineComment  CommentRule = Comment("//", "\n")
	CppBlockComment CommentRule = Comment("/*", "*/")
)

func Comment 

func Comment(begin string, end string) CommentRule

func (CommentRule) Eval 

func (r CommentRule) Eval(s *Scanner) bool

func (CommentRule) WithKeep 

func (r CommentRule) WithKeep(keep *bool) CommentRule

type ExpRule 

type ExpRule struct {
	// contains filtered or unexported fields
}
var (
	Exp    ExpRule = NewExpRule(Rune('E', 'e'), IsSign, IsDigit09)
	HexExp ExpRule = NewExpRule(Rune('P', 'p'), IsSign, IsDigit0F)
)

func NewExpRule 

func NewExpRule(isExp Class, isSign Class, isDigit Class) ExpRule

func (ExpRule) Eval 

func (r ExpRule) Eval(scan *Scanner) bool

func (ExpRule) WithDigitSep 

func (r ExpRule) WithDigitSep(c Class) ExpRule

func (ExpRule) WithEmpty 

func (r ExpRule) WithEmpty(b bool) ExpRule

type FracRule 

type FracRule struct {
	// contains filtered or unexported fields
}
var (
	Frac    FracRule = NewFracRule(IsDigit09, Rune('.'))
	HexFrac FracRule = NewFracRule(IsDigit0F, Rune('.'))
)

func NewFracRule 

func NewFracRule(isDigit Class, isDecSep Class) FracRule

func (FracRule) Eval 

func (r FracRule) Eval(scan *Scanner) bool

func (FracRule) WithDigitSep 

func (r FracRule) WithDigitSep(c Class) FracRule

func (FracRule) WithEmpty 

func (r FracRule) WithEmpty(b bool) FracRule

type IdentRule 

type IdentRule struct {
	// contains filtered or unexported fields
}
var GoIdent IdentRule = Ident(IsLetterUnder, IsLetterDigitUnder)

func Ident 

func Ident(head Class, tail Class) IdentRule

func (IdentRule) Eval 

func (r IdentRule) Eval(s *Scanner) bool

func (IdentRule) WithKeywords 

func (r IdentRule) WithKeywords(ks ...string) IdentRule

type IntRule 

type IntRule struct {
	// contains filtered or unexported fields
}
var (
	Bin  IntRule = NewIntRule(IsDigit01).WithKind(BinKind)
	Hex  IntRule = NewIntRule(IsDigit0F).WithKind(HexKind)
	Int  IntRule = UInt.WithSign(IsSign)
	Oct  IntRule = NewIntRule(IsDigit07).WithKind(OctKind)
	UInt IntRule = NewIntRule(IsDigit09)
)

func NewIntRule 

func NewIntRule(isDigit Class) IntRule

func (IntRule) Eval 

func (r IntRule) Eval(scan *Scanner) bool

func (IntRule) WithDigitSep 

func (r IntRule) WithDigitSep(c Class) IntRule

func (IntRule) WithKind 

func (r IntRule) WithKind(k string) IntRule

func (IntRule) WithLeadingDigitSep 

func (r IntRule) WithLeadingDigitSep(b bool) IntRule

func (IntRule) WithLeadingZero 

func (r IntRule) WithLeadingZero(b bool) IntRule

func (IntRule) WithSign 

func (r IntRule) WithSign(c Class) IntRule

type KindSet 

type KindSet []string

func NewKindSet 

func NewKindSet(kinds ...string) KindSet

func (*KindSet) Add 

func (k *KindSet) Add(kinds ...string)

func (KindSet) Contains 

func (k KindSet) Contains(k2 KindSet) bool

func (KindSet) ContainsAny 

func (k KindSet) ContainsAny(kinds ...string) bool

func (*KindSet) Remove 

func (k *KindSet) Remove(kinds ...string)

func (KindSet) String 

func (k KindSet) String() string

type NumRule 

type NumRule struct {
	// contains filtered or unexported fields
}
var (
	Real        NumRule = NewNumRule(Int, Frac)
	RealExp     NumRule = NewNumRule(Int, Frac).WithExpRule(Exp)
	HexURealExp NumRule = NewNumRule(UInt, HexFrac).WithExpRule(HexExp)
	HexRealExp  NumRule = NewNumRule(Hex, HexFrac).WithExpRule(HexExp)
	UReal       NumRule = NewNumRule(UInt, Frac)
	URealExp    NumRule = UReal.WithExpRule(Exp)
)

func NewNumRule 

func NewNumRule(intRule IntRule, fracRule FracRule) NumRule

func (NumRule) Eval 

func (r NumRule) Eval(scan *Scanner) bool

func (NumRule) WithExpRule 

func (r NumRule) WithExpRule(er ExpRule) NumRule

type Pos 

type Pos struct {
	Name   string `json:",omitempty"`
	Line   int
	Column int
}

Pos represents a position within an input stream.

func NewPos 

func NewPos(name string) Pos

NewPos returns a new position with the given name and the line number and column number set to one.

func (Pos) String 

func (p Pos) String() string

type PrefixRule 

type PrefixRule struct {
	// contains filtered or unexported fields
}

func Prefix 

func Prefix(prefix string, rule Rule) PrefixRule

func (PrefixRule) Eval 

func (r PrefixRule) Eval(s *Scanner) bool

type Rule 

type Rule interface {
	Eval(*Scanner) bool
}

func NewRule 

func NewRule(eval func(*Scanner) bool) Rule

type RuleSet 

type RuleSet struct {
	// contains filtered or unexported fields
}

func Rules 

func Rules(rules ...Rule) RuleSet

func SymbolMap 

func SymbolMap(sk map[string]string) RuleSet

func Symbols 

func Symbols(ss ...string) RuleSet

func (RuleSet) Eval 

func (r RuleSet) Eval(scan *Scanner) bool

func (RuleSet) Next 

func (r RuleSet) Next(scan *Scanner) Token

func (RuleSet) WithDiscards 

func (r RuleSet) WithDiscards(c Class) RuleSet

func (RuleSet) WithPostTokenFunc 

func (r RuleSet) WithPostTokenFunc(fn func(*Scanner, Token) Token) RuleSet

func (RuleSet) WithPreTokenFunc 

func (r RuleSet) WithPreTokenFunc(fn func(*Scanner)) RuleSet

type Runner 

type Runner struct {
	// contains filtered or unexported fields
}

func NewRunner 

func NewRunner(scan *Scanner, rules RuleSet) *Runner

func (*Runner) All 

func (r *Runner) All() []Token

func (*Runner) More 

func (r *Runner) More() bool

func (*Runner) Next 

func (r *Runner) Next() Token

func (*Runner) Peek 

func (r *Runner) Peek() Token

type Scanner 

type Scanner struct {
	Name string
	This rune
	Prev rune
	Kind KindSet
	// contains filtered or unexported fields
}

func New 

func New(src io.Reader) *Scanner

func NewFromBytes 

func NewFromBytes(src []byte) *Scanner

func NewFromString 

func NewFromString(src string) *Scanner

func (*Scanner) Discard 

func (s *Scanner) Discard()

func (*Scanner) Emit 

func (s *Scanner) Emit() Token

Emit returns the token that has been built and resets the builder for the next token.

func (*Scanner) Illegal 

func (s *Scanner) Illegal(format string, args ...any)

func (*Scanner) Init 

func (s *Scanner) Init(src io.Reader)

Init resets the input stream to src.

func (*Scanner) InitFromBytes 

func (s *Scanner) InitFromBytes(src []byte)

InitBytes resets the input stream to src.

func (*Scanner) InitFromString 

func (s *Scanner) InitFromString(src string)

InitString resets the input stream to src.

func (*Scanner) Keep 

func (s *Scanner) Keep()

Keep adds the current rune to the token being built and advances the stream to the next rune.

func (*Scanner) KeepIfPrefix 

func (s *Scanner) KeepIfPrefix(prefix string) bool

func (*Scanner) Lit 

func (s *Scanner) Lit() string

func (*Scanner) More 

func (s *Scanner) More() bool

func (*Scanner) Peek 

func (s *Scanner) Peek(n int) rune

func (*Scanner) PeekTo 

func (s *Scanner) PeekTo(n int) string

func (*Scanner) Skip 

func (s *Scanner) Skip()

Skip discards the current rune and advances the stream to the next rune.

func (*Scanner) Val 

func (s *Scanner) Val() string

func (*Scanner) WriteRune 

func (s *Scanner) WriteRune(r rune)

type StrRule 

type StrRule struct {
	// contains filtered or unexported fields
}

func Str 

func Str(begin rune, end rune) StrRule

func (StrRule) Eval 

func (r StrRule) Eval(s *Scanner) bool

func (StrRule) WithKind 

func (r StrRule) WithKind(k string) StrRule

func (StrRule) WithMultiline 

func (r StrRule) WithMultiline() StrRule

type SuffixRule 

type SuffixRule struct {
	// contains filtered or unexported fields
}

func Suffix 

func Suffix(suffix Class, rule Rule) SuffixRule

func (SuffixRule) Eval 

func (r SuffixRule) Eval(s *Scanner) bool

func (SuffixRule) WithKind 

func (r SuffixRule) WithKind(k string) SuffixRule

type SymbolRule 

type SymbolRule struct {
	// contains filtered or unexported fields
}

func Symbol 

func Symbol(symbol string) SymbolRule

func (SymbolRule) Eval 

func (r SymbolRule) Eval(s *Scanner) bool

func (SymbolRule) WithKind 

func (r SymbolRule) WithKind(k string) SymbolRule

type TestCase 

type TestCase struct {
	// contains filtered or unexported fields
}

func NewTestCase 

func NewTestCase(src string, val string, kinds ...string) TestCase

func (TestCase) And 

func (t TestCase) And(val string, kinds ...string) TestCase

type Token 

type Token struct {
	Val  string
	Lit  string `json:",omitempty"`
	Kind KindSet
	Pos  Pos
	Err  TokenError
}

func (Token) IsKind 

func (t Token) IsKind(k ...string) bool

func (Token) String 

func (t Token) String() string

type TokenError 

type TokenError interface {
	Pos() Pos
	Message() string
	Error() string
	Unwrap() error
}

Source Files

View all Source files

Directories

Path Synopsis
examples
dms
go
json
json-ez
internal
scan-go

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.