lexer

Documentation

Overview

Package lexer provides basic helpers to implement parsers

Index

• Constants
• Variables
• func IsSpace(r rune) bool
• func NewIsIn(s string) func(rune) bool
• func NewIsInRunes(s ...rune) func(rune) bool
• func NewIsNot(cond func(rune) bool) func(rune) bool
• func NewIsOneOf(s ...func(rune) bool) func(rune) bool
• func Run(fn StateFn) error
• type Error
  • func (err Error) Error() string
  • func (err Error) Unwrap() error
• type Position
  • func (p *Position) Add(rel Position)
  • func (p Position) GoString() string
  • func (p *Position) Reset()
  • func (p *Position) Step()
  • func (p *Position) StepLine()
  • func (p *Position) StepN(n int)
  • func (p Position) String() string
• type Reader
  • func NewReader(r io.Reader) *Reader
  • func NewReaderBytes(b []byte) *Reader
  • func NewReaderString(s string) *Reader
  • func (b *Reader) Accept(cond func(r rune) bool) bool
  • func (b *Reader) AcceptAll(cond func(r rune) bool) bool
  • func (b *Reader) Discard()
  • func (b *Reader) Emit() string
  • func (b *Reader) PeekRune() (rune, int, error)
  • func (b *Reader) ReadRune() (rune, int, error)
  • func (b *Reader) String() string
  • func (b *Reader) UnreadRune() error
• type StateFn

Constants

const (
	// ReadBufferSize indicates the initial buffer size
	ReadBufferSize = 1 << 7 // 128B

	// DoublingBufferSizeLimit indicates when we stop doubling
	// and just add instead
	DoublingBufferSizeLimit = 1 << 17 // 128KiB
)

Variables

var (
	// ErrUnacceptableRune indicates the read rune isn't acceptable in the context
	ErrUnacceptableRune = errors.New("rune not acceptable in context")

	// ErrNotImplemented indicates something hasn't been implemented yet
	ErrNotImplemented = errors.New("not implemented")
)

var (
	// ErrInvalidUnreadRune indicates UnreadRune() was calls after an
	// action other than a successful ReadRune()
	ErrInvalidUnreadRune = errors.New("invalid UnreadRune() call")
)

Functions

func IsSpace

func IsSpace(r rune) bool

IsSpace reports whether the rune is a space character as defined by Unicode's White Space property

func NewIsIn

func NewIsIn(s string) func(rune) bool

NewIsIn generates a rune condition checker that accepts runes contained on the provided string

func NewIsInRunes

func NewIsInRunes(s ...rune) func(rune) bool

NewIsInRunes generates a rune condition checker that accepts the runes specified

func NewIsNot

func NewIsNot(cond func(rune) bool) func(rune) bool

NewIsNot generates a rune condition checker that reverses the decision of the given checker.

func NewIsOneOf

func NewIsOneOf(s ...func(rune) bool) func(rune) bool

NewIsOneOf generates a run condition checker that accepts runes accepted by any of the given checkers

func Run

func Run(fn StateFn) error

Run runs a state machine until the state function either returns nil or an error

Types

type Error

type Error struct {
	Filename string
	Line     int
	Column   int

	Content string
	Hint    string
	Err     error
}

Error represents a generic parsing error

func (Error) Error

func (err Error) Error() string

func (Error) Unwrap

func (err Error) Unwrap() error

type Position

type Position struct {
	Line   int
	Column int
}

Position indicates a line and column pair on a file. Counting starts at 1.

func (*Position) Add

func (p *Position) Add(rel Position)

Add adds a relative position considering potential new lines

func (Position) GoString

func (p Position) GoString() string

GoString generates a string representation of the Position for %#v usage

func (*Position) Reset

func (p *Position) Reset()

Reset places a position at (1,1)

func (*Position) Step

func (p *Position) Step()

Step moves the column one place

func (*Position) StepLine

func (p *Position) StepLine()

StepLine moves position to the start of the next line

func (*Position) StepN

func (p *Position) StepN(n int)

StepN moves the column N places forward

func (Position) String

func (p Position) String() string

String generates a pretty "(Line, Column)"" representation of the Position

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader is a RuneReader aimed at implementing text parsers

func NewReader

func NewReader(r io.Reader) *Reader

NewReader creates a new runes Reader using the given io.Reader

func NewReaderBytes

func NewReaderBytes(b []byte) *Reader

NewReaderBytes creates a new runes Reader using the given bytes

func NewReaderString

func NewReaderString(s string) *Reader

NewReaderString creates a new runes Reader using the given string

func (*Reader) Accept

func (b *Reader) Accept(cond func(r rune) bool) bool

Accept consumes a rune from the source if it meets the condition. it returns true if the condition was met and false if it wasn't.

func (*Reader) AcceptAll

func (b *Reader) AcceptAll(cond func(r rune) bool) bool

AcceptAll consumes runes from the source as long as they meet the condition. it returns true if the condition was met for at least one rune, and false if it wasn't.

func (*Reader) Discard

func (b *Reader) Discard()

Discard removes from the buffer everything that has been Read

func (*Reader) Emit

func (b *Reader) Emit() string

Emit returns what's already being Read and discards it afterwards

func (*Reader) PeekRune

func (b *Reader) PeekRune() (rune, int, error)

PeekRune returns information about the next rune without moving the cursor

func (*Reader) ReadRune

func (b *Reader) ReadRune() (rune, int, error)

ReadRune reads the next rune

func (*Reader) String

func (b *Reader) String() string

String returns what's already Read but not yet emitted or discarded

func (*Reader) UnreadRune

func (b *Reader) UnreadRune() error

UnreadRune moves the cursor where it was before the last call to ReadRune

type StateFn

type StateFn func() (StateFn, error)

StateFn is a State Function of the parser