Versions in this module Expand all Collapse all v1 v1.1.0 Oct 17, 2018 Changes in this version type MaxMatch + EnabledFilterStopToken bool + StopTokens *StopTokens + WordFilter WordFilter + type NumAndLetterWordFilter struct + func (nlFilter *NumAndLetterWordFilter) Filter(text string) bool type ReverseMaxMatch + EnabledFilterStopToken bool + StopTokens *StopTokens + WordFilter WordFilter + type WordFilter interface + Filter func(text string) bool v1.0.0 Oct 15, 2018 Changes in this version + var DefaultMinTokenLen = 2 + func CheckDictIsLoaded(dict *Dict) error + func GetFrequency(result []string) map[string]int + func Reverse(s []string) []string + type BiDirectionalMaxMatch struct + MM *MaxMatch + MMScore float64 + RMM *ReverseMaxMatch + RMMScore float64 + func NewBiDirectionalMaxMatch(dictPath, bigramDictPath string) *BiDirectionalMaxMatch + func (bdmm *BiDirectionalMaxMatch) Get(text string) ([]string, error) + func (bdmm *BiDirectionalMaxMatch) GetFrequency(text string) (map[string]int, error) + func (bdmm *BiDirectionalMaxMatch) LoadDict() error + type BiDirectionalMinMatch struct + MM *MinMatch + MMScore float64 + RMM *ReverseMinMatch + RMMScore float64 + func NewBiDirectionalMinMatch(dictPath, bigramDictPath string) *BiDirectionalMinMatch + func (bdmm *BiDirectionalMinMatch) Get(text string) ([]string, error) + func (bdmm *BiDirectionalMinMatch) GetFrequency(text string) (map[string]int, error) + func (bdmm *BiDirectionalMinMatch) LoadDict() error + type BigramDict struct + func NewBigramDict(dictPath string) *BigramDict + func (bd *BigramDict) Load() error + type Dict struct + DictPath string + Records map[string]DictRecord + func NewDict(dictPath string) *Dict + func (dict *Dict) Load() error + type DictRecord struct + POS string + TF string + Token string + type MaxMatch struct + func NewMaxMatch(dictPath string) *MaxMatch + func (mm *MaxMatch) Get(text string) ([]string, error) + func (mm *MaxMatch) GetFrequency(text string) (map[string]int, error) + func (mm *MaxMatch) LoadDict() error + type MinMatch struct + func NewMinMatch(dictPath string) *MinMatch + func (mm *MinMatch) Get(text string) ([]string, error) + func (mm *MinMatch) GetFrequency(text string) (map[string]int, error) + func (mm *MinMatch) LoadDict() error + type ReverseMaxMatch struct + func NewReverseMaxMatch(dictPath string) *ReverseMaxMatch + func (rmm *ReverseMaxMatch) Get(text string) ([]string, error) + func (rmm *ReverseMaxMatch) GetFrequency(text string) (map[string]int, error) + func (rmm *ReverseMaxMatch) LoadDict() error + type ReverseMinMatch struct + func NewReverseMinMatch(dictPath string) *ReverseMinMatch + func (rmm *ReverseMinMatch) Get(text string) ([]string, error) + func (rmm *ReverseMinMatch) GetFrequency(text string) (map[string]int, error) + func (rmm *ReverseMinMatch) LoadDict() error + type StopTokens struct + IsLoaded bool + func NewStopTokens() *StopTokens + func (st *StopTokens) IsStopToken(token string) bool + func (st *StopTokens) Load(path string) error + type Tokenizer interface + Get func(text string) ([]string, error) + GetFrequency func(text string) (map[string]int, error) + LoadDict func() error