lexer

package
Version: v0.0.0-...-c1b4264 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 14, 2021 License: GPL-3.0 Imports: 3 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var ParseCharLiteral = ToParser("ParseCharLiteral", func(s *TextScanner) []Token {
	chars := make([]rune, 0)
	position := s.Position()
	c := s.Current()
	if c != '\'' {
		return nil
	}
	chars = append(chars, c)
	c2 := s.Next()
	if c2 == '\\' {
		c3 := s.Next()
		c4 := s.Next()
		if c4 != '\'' {
			return nil
		}
		chars = append(chars, c2, c3, c4)
	} else {
		c3 := s.Next()
		if c3 != '\'' {
			return nil
		}
		chars = append(chars, c2, c3)
	}
	s.Next()
	return []Token{{TokenType: "CharLiteral", TokenText: string(chars), Position: position}}
})

解析字符字面量

View Source
var ParseFloatLiteral = ToParser("ParseFloatLiteral", func(s *TextScanner) []Token {
	chars := make([]rune, 0)
	position := s.Position()
	c := s.Current()
	if !IsDigit(c) {
		return nil
	}
	chars = append(chars, c)
	hasPoint := false
	for {
		c := s.Next()
		if IsDigit(c) || c == '.' || c == '_' || c == 'e' || c == 'E' {
			chars = append(chars, c)
			if c == '.' {
				hasPoint = true
			}
		} else {
			break
		}
	}
	if !hasPoint {
		return nil
	}
	return []Token{{TokenType: "FloatLiteral", TokenText: string(chars), Position: position}}
})

解析浮点数字面量

View Source
var ParseID = ToParser("ParseID", func(s *TextScanner) []Token {
	chars := make([]rune, 0)
	position := s.Position()
	c := s.Current()
	if !IsAlpha(c) {
		return nil
	}
	chars = append(chars, c)
	for {
		c := s.Next()
		if IsAlpha(c) || IsDigit(c) || c == '_' {
			chars = append(chars, c)
		} else {
			break
		}
	}
	return []Token{{TokenType: "ID", TokenText: string(chars), Position: position}}
})

解析标志符

View Source
var ParseIntLiteral = ToParser("ParseIntLiteral", func(s *TextScanner) []Token {
	chars := make([]rune, 0)
	position := s.Position()
	c := s.Current()
	if !IsDigit(c) {
		return nil
	}
	chars = append(chars, c)
	for {
		c := s.Next()
		if IsDigit(c) || c == '_' || c == 'e' || c == 'E' {
			chars = append(chars, c)
		} else {
			break
		}
	}
	return []Token{{TokenType: "IntLiteral", TokenText: string(chars), Position: position}}
})

解析整数字面量

View Source
var ParseNumberLiteral = OneOf(ParseFloatLiteral, ParseIntLiteral)

解析数值字面量

View Source
var ParseStringLiteral = ToParser("ParseStringLiteral", func(s *TextScanner) []Token {
	chars := make([]rune, 0)
	position := s.Position()
	c := s.Current()
	if c != '"' {
		return nil
	}
	chars = append(chars, c)
	lc := c
	for {
		c := s.Next()
		if c == '"' && lc != '\\' {
			chars = append(chars, c)
			s.Next()
			break
		} else {
			chars = append(chars, c)
			lc = c
		}
	}
	return []Token{{TokenType: "StringLiteral", TokenText: string(chars), Position: position}}
})

解析字符串字面量

View Source
var ParseWhite = ToParser("ParseWhite", func(s *TextScanner) []Token {
	chars := make([]rune, 0)
	position := s.Position()
	c := s.Current()
	if !IsWhite(c) {
		return nil
	}
	chars = append(chars, c)
	for {
		c := s.Next()
		if IsWhite(c) {
			chars = append(chars, c)
		} else {
			break
		}
	}
	return []Token{{TokenType: "White", TokenText: string(chars), Position: position}}
})

解析空白字符

Functions

func Debug

func Debug(enable bool, slowdown time.Duration)

func IsAlpha

func IsAlpha(c rune) bool

是否为字母

func IsDigit

func IsDigit(c rune) bool

是否为数字

func IsInStringArray

func IsInStringArray(arr []string, s string) bool

字符串是否在数组内

func IsWhite

func IsWhite(c rune) bool

是否为空白字符

Types

type Parser

type Parser struct {
	Name string
	Func ParserFunc
}

func DiscardResult

func DiscardResult(parser *Parser) *Parser

丢弃解析到的结果

func GenParseKeyword

func GenParseKeyword(keywords []string) *Parser

解析关键词

func GenParseToken

func GenParseToken(tokenType string, text string) *Parser

生成解析固定连续字符

func Many

func Many(parser *Parser) *Parser

func OneOf

func OneOf(parsers ...*Parser) *Parser

尝试执行任意一个解析器,如果成功则返回

func OneOrMore

func OneOrMore(parser *Parser) *Parser

重复一个或多个的解析器

func Optional

func Optional(parser *Parser) *Parser

尝试可选的解析器

func Sequence

func Sequence(parsers ...*Parser) *Parser

尝试顺序执行解析器

func SequenceWithOptionalWhitespace

func SequenceWithOptionalWhitespace(parsers ...*Parser) *Parser

尝试顺序执行解析器,每个解析器中间加入可选的空白字符解析

func ToParser

func ToParser(name string, parser ParserFunc) *Parser

func ZeroOrMore

func ZeroOrMore(parser *Parser) *Parser

重复零个或多个的解析器

func (*Parser) Call

func (p *Parser) Call(s *TextScanner) []Token

func (*Parser) SetName

func (p *Parser) SetName(name string) *Parser

type ParserFunc

type ParserFunc = func(s *TextScanner) []Token

type SourcePosition

type SourcePosition struct {
	Offset int
	Line   int
	Column int
}

type TextScanner

type TextScanner struct {
	// contains filtered or unexported fields
}

func NewTextScanner

func NewTextScanner(source string) *TextScanner

func (*TextScanner) Current

func (s *TextScanner) Current() rune

当前字符

func (*TextScanner) EOF

func (s *TextScanner) EOF() bool

是否到末尾

func (*TextScanner) GetLine

func (s *TextScanner) GetLine(offset int) (line string, start int, end int)

获得指定位置所在行的内容

func (*TextScanner) Next

func (s *TextScanner) Next() rune

移到下一个字符

func (*TextScanner) Offset

func (s *TextScanner) Offset() int

当前位置

func (*TextScanner) Peek

func (s *TextScanner) Peek() rune

偷看下一个字符

func (*TextScanner) Position

func (s *TextScanner) Position() SourcePosition

当前位置信息

func (*TextScanner) StackPop

func (s *TextScanner) StackPop(reset bool)

从栈中弹出位置 reset 如果为true则根据栈顶的信息重置当前位置

func (*TextScanner) StackPush

func (s *TextScanner) StackPush()

将当前位置压栈

type Token

type Token struct {
	TokenType string         // 符号类型
	TokenText string         // 符号文本
	Position  SourcePosition // 开始位置
}

func RunParser

func RunParser(parser *Parser, scanner *TextScanner) ([]Token, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
t or T : Toggle theme light dark auto
y or Y : Canonical URL