tokenizer

Oct 31, 2022

Changes in this version

type Token

+ func (t Token) UserExtra() *UserExtra

+ type UserExtra struct

+ Readings []string

+ Tokens []string

Jun 20, 2022

Changes in this version

+ func EqualFeatures(lhs, rhs []string) bool

type Token

+ func (t Token) EqualFeatures(tt Token) bool

+ func (t Token) EqualPOS(tt Token) bool

Sep 5, 2021

Changes in this version

+ type TokenData struct

+ BaseForm string

+ Class string

+ End int

+ Features []string

+ ID int

+ POS []string

+ Pronunciation string

+ Reading string

+ Start int

+ Surface string

+ func NewTokenData(t Token) TokenData

Dec 29, 2020

Changes in this version

type Token

+ Index int

Oct 10, 2020

Changes in this version

type Option

+ func OmitBosEos() Option

type Token

+ Position int

+ func (t Token) Equal(v Token) bool

Aug 9, 2020

Changes in this version

+ const DUMMY

+ const KNOWN

+ const UNKNOWN

+ const USER

+ type Option func(*Tokenizer) error

+ func Nop() Option

+ func UserDict(d *dict.UserDict) Option

+ type Token struct

+ Class TokenClass

+ End int

+ ID int

+ Start int

+ Surface string

+ func (t Token) BaseForm() (string, bool)

+ func (t Token) FeatureAt(i int) (string, bool)

+ func (t Token) Features() []string

+ func (t Token) InflectionalForm() (string, bool)

+ func (t Token) InflectionalType() (string, bool)

+ func (t Token) POS() []string

+ func (t Token) Pronunciation() (string, bool)

+ func (t Token) Reading() (string, bool)

+ func (t Token) String() string

+ type TokenClass lattice.NodeClass

+ func (c TokenClass) String() string

+ type TokenizeMode int

+ const BosEosID

+ const Extended

+ const Normal

+ const Search

+ func (m TokenizeMode) String() string

+ type Tokenizer struct

+ func New(d *dict.Dict, opts ...Option) (*Tokenizer, error)

+ func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token)

+ func (t Tokenizer) AnalyzeGraph(w io.Writer, input string, mode TokenizeMode) (tokens []Token)

+ func (t Tokenizer) Dot(w io.Writer, input string) (tokens []Token)

+ func (t Tokenizer) Tokenize(input string) []Token

+ func (t Tokenizer) Wakati(input string) []string