Versions in this module Expand all Collapse all v0 v0.66.9 Apr 7, 2021 Changes in this version + const RatioWord + const RatioWordFull + const Version + var StopWordMap = map[string]bool + var ToLower = true + func DictPaths(dictDir, filePath string) (files []string) + func FilterEmoji(text string) (new string) + func FilterHtml(text string) string + func FilterLang(text, lang string) (new string) + func FilterSymbol(text string) (new string) + func GetCurrentFilePath() string + func GetIdfPath(files ...string) []string + func GetVersion() string + func IsJp(segText string) bool + func Join(text []Text) string + func Range(text string) (new []string) + func RangeText(text string) (new string) + func SplitNum(text string) []string + func SplitNums(text string) string + func ToSlice(segs []Segment, searchMode ...bool) (output []string) + func ToString(segs []Segment, searchMode ...bool) (output string) + type Dictionary struct + Tokens []Token + func NewDict() *Dictionary + func (dict *Dictionary) Find(word []byte) (float64, bool) + func (dict *Dictionary) LookupTokens(words []Text, tokens []*Token) (numOfTokens int) + func (dict *Dictionary) MaxTokenLen() int + func (dict *Dictionary) NumTokens() int + func (dict *Dictionary) RemoveToken(token Token) error + func (dict *Dictionary) TotalFreq() float64 + func (dict *Dictionary) Value(word []byte) (val, id int, err error) + type Prob struct + B map[rune]float64 + E map[rune]float64 + M map[rune]float64 + S map[rune]float64 + type SegPos struct + Pos string + Text string + func ToPos(segs []Segment, searchMode ...bool) (output []SegPos) + type Segment struct + Position int + func (s *Segment) End() int + func (s *Segment) Start() int + func (s *Segment) Token() *Token + type Segmenter struct + Alpha bool + AlphaNum bool + Dict *Dictionary + Load bool + LoadNoFreq bool + MinTokenFreq float64 + MoreLog bool + NotStop bool + Num bool + SkipLog bool + SkipPos bool + StopWordMap map[string]bool + TextFreq string + func New(files ...string) Segmenter + func NewEmbed(alpha ...string) (seg Segmenter) + func (seg *Segmenter) AddStop(text string) + func (seg *Segmenter) AddToken(text string, frequency float64, pos ...string) error + func (seg *Segmenter) AddTokenForce(text string, frequency float64, pos ...string) + func (seg *Segmenter) CalcToken() + func (seg *Segmenter) Cut(str string, hmm ...bool) []string + func (seg *Segmenter) CutAll(str string) []string + func (seg *Segmenter) CutSearch(str string, hmm ...bool) []string + func (seg *Segmenter) CutStr(str []string, separator ...string) (r string) + func (seg *Segmenter) CutTrim(str string, hmm ...bool) []string + func (seg *Segmenter) CutTrimHtml(str string, hmm ...bool) []string + func (seg *Segmenter) CutTrimHtmls(str string, hmm ...bool) string + func (seg *Segmenter) CutUrl(str string, num ...bool) []string + func (seg *Segmenter) CutUrls(str string, num ...bool) string + func (seg *Segmenter) Dictionary() *Dictionary + func (seg *Segmenter) Find(str string) (float64, bool) + func (seg *Segmenter) HMMCut(str string) []string + func (seg *Segmenter) HMMCutMod(str string, prob ...map[rune]float64) []string + func (seg *Segmenter) Init() + func (seg *Segmenter) IsStop(s string) bool + func (seg *Segmenter) LoadDict(files ...string) error + func (seg *Segmenter) LoadDictByCustomizedPath(files []string) error + func (seg *Segmenter) LoadDictEmbed() error + func (seg *Segmenter) LoadDictMap(dict []map[string]string) error + func (seg *Segmenter) LoadDictStr(dict string) error + func (seg *Segmenter) LoadModel(prob ...map[rune]float64) + func (seg *Segmenter) LoadStop(files ...string) error + func (seg *Segmenter) LoadStopArr(dict []string) + func (seg *Segmenter) LoadStopEmbed() error + func (seg *Segmenter) ModeSegment(bytes []byte, searchMode ...bool) []Segment + func (seg *Segmenter) Pos(s string, searchMode ...bool) []SegPos + func (seg *Segmenter) PosStr(str []SegPos, separator ...string) (r string) + func (seg *Segmenter) PosTrim(str string, search bool, pos ...string) []SegPos + func (seg *Segmenter) PosTrimArr(str string, search bool, pos ...string) (re []string) + func (seg *Segmenter) PosTrimStr(str string, search bool, pos ...string) string + func (seg *Segmenter) Read(file string) error + func (seg *Segmenter) Reader(reader io.Reader, files ...string) error + func (seg *Segmenter) RemoveStop(text string) + func (seg *Segmenter) RemoveToken(text string) error + func (seg *Segmenter) Segment(bytes []byte) []Segment + func (seg *Segmenter) Size(size int, text, freqText string) (frequency float64) + func (seg *Segmenter) Slice(s string, searchMode ...bool) []string + func (seg *Segmenter) SplitTextToWords(text Text) []Text + func (seg *Segmenter) String(s string, searchMode ...bool) string + func (seg *Segmenter) SuggestFreq(words ...string) float64 + func (seg *Segmenter) Trim(s []string) (r []string) + func (seg *Segmenter) TrimPos(s []SegPos) (r []SegPos) + func (seg *Segmenter) TrimPosPunct(se []SegPos) (re []SegPos) + func (seg *Segmenter) TrimPunct(s []string) (r []string) + func (seg *Segmenter) TrimSymbol(s []string) (r []string) + func (seg *Segmenter) TrimWithPos(se []SegPos, pos ...string) (re []SegPos) + func (seg *Segmenter) Value(str string) (int, int, error) + type Text []byte + func SplitWords(text Text) []Text + type Token struct + func (token *Token) Equals(str string) bool + func (token *Token) Frequency() float64 + func (token *Token) Pos() string + func (token *Token) Segments() []*Segment + func (token *Token) Text() string
Go to main page