cjk

Documentation

Index

• Variables
• func NewBigramFilter(outputUnigram bool) analysis.TokenFilter
• func WidthFilter(input analysis.TokenStream) analysis.TokenStream
• type BigramFilter
  • func (s *BigramFilter) Filter(input analysis.TokenStream) analysis.TokenStream

Variables

var Analyzer = &analysis.Analyzer{
	Tokenizer: tokenizer.UnicodeTokenizer,
	TokenFilters: []analysis.TokenFilter{
		WidthFilter,
		token.DefaultLowerCaseFilter,
		DefaultBigramFilter,
	},
}

var DefaultBigramFilter = NewBigramFilter(false)

Functions

func NewBigramFilter

func NewBigramFilter(outputUnigram bool) analysis.TokenFilter

func WidthFilter

func WidthFilter(input analysis.TokenStream) analysis.TokenStream

Types

type BigramFilter

type BigramFilter struct {
	// contains filtered or unexported fields
}

func (*BigramFilter) Filter

func (s *BigramFilter) Filter(input analysis.TokenStream) analysis.TokenStream