wikitext

Documentation

Index

• func ParseDefinition(raw string, depth int) (string, error)
• func ParseWord(raw string) string
• type State
• type StateStack
• type Token
• type TokenCollection
  • func (t *TokenCollection) Stringify() (string, error)
• type TokenGrammar
• type TokenType
• type Tokenizer
  • func NewTokenizer(raw string) Tokenizer
  • func (t *Tokenizer) GetTokenType() TokenGrammar
  • func (t *Tokenizer) Tokenize(options TokenizerOptions) *Tokenizer
• type TokenizerOptions

Functions

func ParseDefinition

func ParseDefinition(raw string, depth int) (string, error)

func ParseWord

func ParseWord(raw string) string

Types

type State

type State string

const (
	EOF State = "EOF"
)

type StateStack

type StateStack []State

type Token

type Token struct {
	Type  State
	Value []byte
}

type TokenCollection

type TokenCollection []Token

Since we only care about text content, tokenized results are represented as a flat collection.

func (*TokenCollection) Stringify

func (t *TokenCollection) Stringify() (string, error)

Used for testing purposes

type TokenGrammar

type TokenGrammar struct {
	Token TokenType
	State State
}

var TEXT_TOKEN TokenGrammar = TokenGrammar{Token: text_token, State: text}

type TokenType

type TokenType string

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

func NewTokenizer

func NewTokenizer(raw string) Tokenizer

func (*Tokenizer) GetTokenType

func (t *Tokenizer) GetTokenType() TokenGrammar

func (*Tokenizer) Tokenize

func (t *Tokenizer) Tokenize(options TokenizerOptions) *Tokenizer

FIXME: this can be cleaned up, let's make it leaner and easier to extend -- already started the improvements with the tokentype func, let's keep going

type TokenizerOptions

type TokenizerOptions struct {
	// contains filtered or unexported fields
}