tokenizers

Apr 16, 2024

Changes in this version

+ type TokenizerOption func(to *tokenizerOpts)

+ func WithEncodeSpecialTokens() TokenizerOption

Feb 9, 2024

Changes in this version

+ type EncodeOption func(eo *EncodeOptions)

+ func WithReturnAllAttributes() EncodeOption

+ func WithReturnAttentionMask() EncodeOption

+ func WithReturnOffsets() EncodeOption

+ func WithReturnSpecialTokensMask() EncodeOption

+ func WithReturnTokens() EncodeOption

+ func WithReturnTypeIDs() EncodeOption

+ type EncodeOptions struct

+ AddSpecialTokens C.bool

+ ReturnAttentionMask C.bool

+ ReturnOffsets C.bool

+ ReturnSpecialTokensMask C.bool

+ ReturnTokens C.bool

+ ReturnTypeIDs C.bool

+ type Encoding struct

+ AttentionMask []uint32

+ IDs []uint32

+ Offsets []Offset

+ SpecialTokensMask []uint32

+ Tokens []string

+ TypeIDs []uint32

+ type Offset [2]uint

+ type Tokenizer struct

+ func FromBytes(data []byte) (*Tokenizer, error)

+ func FromBytesWithTruncation(data []byte, maxLen uint32, dir TruncationDirection) (*Tokenizer, error)

+ func FromFile(path string) (*Tokenizer, error)

+ func (t *Tokenizer) Close() error

+ func (t *Tokenizer) Decode(tokenIDs []uint32, skipSpecialTokens bool) string

+ func (t *Tokenizer) Encode(str string, addSpecialTokens bool) ([]uint32, []string)

+ func (t *Tokenizer) EncodeWithOptions(str string, addSpecialTokens bool, opts ...EncodeOption) Encoding

+ func (t *Tokenizer) VocabSize() uint32

+ type TruncationDirection int

+ const TruncationDirectionLeft

+ const TruncationDirectionRight