Versions in this module Expand all Collapse all v1 v1.0.1 Apr 8, 2023 v1.0.0 Apr 8, 2023 Changes in this version + const ENDOFPROMPT + const ENDOFTEXT + const FIM_MIDDLE + const FIM_PREFIX + const FIM_SUFFIX + var ENCODING_MAP = map[string]*Encoding + var MODEL_TO_ENCODING = map[string]string + type CoreBPE struct + func NewCoreBPE(encoder map[string]int, specialTokensEncoder map[string]int, pattern string) (*CoreBPE, error) + type Encoding struct + ExplicitNVocab int + MergeableRanks map[string]int + Name string + PatStr string + SpecialTokens map[string]int + type Tiktoken struct + func EncodingForModel(modelName string) (*Tiktoken, error) + func GetEncoding(encodingName string) (*Tiktoken, error) + func (t *Tiktoken) Decode(tokens []int) string + func (t *Tiktoken) Encode(text string, allowedSpecial []string, disallowedSpecial []string) []int + func (t *Tiktoken) SpecialTokenRegex(disallowedSpecialSet map[string]any) *regexp2.Regexp
Go to main page