ybase

README

ybase

Utilities to implement a lexer for goyacc.

Example

package main

import (
	"bytes"
	"fmt"
	"unicode"

	"github.com/berquerant/ybase"
)

func main() {
    input := "1 + 12 - (34-56)"
	s := ybase.NewLexer(ybase.NewScanner(ybase.NewReader(bytes.NewBufferString(input), nil), func(r ybase.Reader) int {
		r.DiscardWhile(unicode.IsSpace)
		top := r.Peek()
		switch {
		case unicode.IsDigit(top):
			r.NextWhile(unicode.IsDigit)
			return 901
		default:
			switch top {
			case '+':
				_ = r.Next()
				return 911
			case '-':
				_ = r.Next()
				return 912
			case '(':
				_ = r.Next()
				return 921
			case ')':
				_ = r.Next()
				return 922
			}
		}
		return ybase.EOF
	}))
	for s.DoLex(func(tok ybase.Token) { fmt.Printf("%d %s\n", tok.Type(), tok.Value()) }) != ybase.EOF {
	}
	if err := s.Err(); err != nil {
		panic(err)
	}
	// Output:
	// 901 1
	// 911 +
	// 901 12
	// 912 -
	// 921 (
	// 901 34
	// 912 -
	// 901 56
	// 922 )
}

Documentation

Index

• Constants
• func NilDebugFunc(format string, v ...any)
• type DebugFunc
• type Lexer
  • func NewLexer(scanner Scanner) Lexer
• type Pos
  • func NewPos(line, col, offset int) Pos
• type Reader
  • func NewReader(rdr io.Reader, debugFunc DebugFunc) Reader
  • func NewReaderWithInitPos(rdr io.Reader, debugFunc DebugFunc, initPos Pos) Reader
• type ScanFunc
• type Scanner
  • func NewScanner(rdr Reader, scanFunc ScanFunc) Scanner
• type Token
  • func NewToken(t int, v string) Token

Examples

• Lexer

Constants

const EOF = -1

Functions

func NilDebugFunc

func NilDebugFunc(format string, v ...any)

Types

type DebugFunc

type DebugFunc func(format string, v ...any)

Debugf outputs debug logs.

type Lexer

type Lexer interface {
	Scanner
	// DoLex runs the lexical analysis.
	// Returns EOF if EOF or an error occurs.
	DoLex(callback func(Token)) int
}

Lexer is an utility to implement yyLexer.

Recommendation: - Set level to yyDebug (YYDEBUG in yacc). - Set yyErrorVerbose to true (YYERROR_VERBOSE in yacc)

Implements yyLexer by Error(string) and Lex(*yySymType) int, e.g.

type ActualLexer struct {
  Lexer
}

func (a *ActualLexer) Lex(lval *yySymType) int {
  return a.DoLex(func(tok Token) {
    lval.token = tok  // declares in %union
  })
}

Example

package main

import (
	"bytes"
	"fmt"
	"unicode"

	"github.com/berquerant/ybase"
)

func main() {
	input := "1 + 12 - (34-56)"
	s := ybase.NewLexer(ybase.NewScanner(ybase.NewReader(bytes.NewBufferString(input), nil), func(r ybase.Reader) int {
		r.DiscardWhile(unicode.IsSpace)
		top := r.Peek()
		switch {
		case unicode.IsDigit(top):
			r.NextWhile(unicode.IsDigit)
			return 901
		default:
			switch top {
			case '+':
				_ = r.Next()
				return 911
			case '-':
				_ = r.Next()
				return 912
			case '(':
				_ = r.Next()
				return 921
			case ')':
				_ = r.Next()
				return 922
			}
		}
		return ybase.EOF
	}))
	for s.DoLex(func(tok ybase.Token) { fmt.Printf("%d %s\n", tok.Type(), tok.Value()) }) != ybase.EOF {
	}
	if err := s.Err(); err != nil {
		panic(err)
	}
}

Output:

901 1
911 +
901 12
912 -
921 (
901 34
912 -
901 56
922 )

func NewLexer

func NewLexer(scanner Scanner) Lexer

type Pos

type Pos interface {
	Line() int
	Column() int
	Offset() int
	Add(r rune) Pos
}

func NewPos

func NewPos(line, col, offset int) Pos

type Reader

type Reader interface {
	// ResetBuffer clears the buffer.
	ResetBuffer()
	// Buffer returns the read runes.
	Buffer() string
	// Next gets the next rune and advances the pos.
	Next() rune
	// Peek gets the next rune but keeps the pos.
	Peek() rune
	// Discard ignores the next rune.
	Discard() rune
	// Err returns an error during the reading.
	Err() error
	// Debugf outputs debug logs.
	Debugf(format string, v ...any)
	// Errorf outputs logs and set an error.
	Errorf(format string, v ...any)
	// DiscardWhile calls Discard() while pred(Peek()).
	DiscardWhile(pred func(rune) bool)
	// NextWhile calls Next() while pred(Peek()).
	NextWhile(pred func(rune) bool)
}

Reader represents a reader object for lex.

func NewReader

func NewReader(rdr io.Reader, debugFunc DebugFunc) Reader

func NewReaderWithInitPos

func NewReaderWithInitPos(rdr io.Reader, debugFunc DebugFunc, initPos Pos) Reader

type ScanFunc

type ScanFunc func(Reader) int

ScanFunc scans source and calculate token.

type Scanner

type Scanner interface {
	Reader
	Scan() int
	// Error consumes an error from yyLexer.
	Error(msg string)
}

func NewScanner

func NewScanner(rdr Reader, scanFunc ScanFunc) Scanner

type Token

type Token interface {
	Type() int
	Value() string
}

func NewToken

func NewToken(t int, v string) Token