single_token

Documentation

Index

• Constants
• func SingleTokenTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error)
• type SingleTokenTokenizer
  • func NewSingleTokenTokenizer() *SingleTokenTokenizer
  • func (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream

Constants

const Name = "single"

Functions

func SingleTokenTokenizerConstructor

func SingleTokenTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error)

Types

type SingleTokenTokenizer

type SingleTokenTokenizer struct {
}

func NewSingleTokenTokenizer

func NewSingleTokenTokenizer() *SingleTokenTokenizer

func (*SingleTokenTokenizer) Tokenize

func (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream