Versions in this module Expand all Collapse all v0 v0.1.2 Jun 10, 2025 v0.1.0 Jun 10, 2025 Changes in this version + var ADA = ModelType + var BABBAGE = ModelType + var CODE_CUSHMAN_001 = ModelType + var CODE_CUSHMAN_002 = ModelType + var CODE_DAVINCI_001 = ModelType + var CODE_DAVINCI_002 = ModelType + var CODE_DAVINCI_EDIT_001 = ModelType + var CODE_SEARCH_ADA_CODE_001 = ModelType + var CODE_SEARCH_BABBAGE_CODE_001 = ModelType + var CURIE = ModelType + var CUSHMAN_CODEX = ModelType + var DAVINCI = ModelType + var DAVINCI_CODEX = ModelType + var GPT_3_5_TURBO = ModelType + var GPT_3_5_TURBO_16K = ModelType + var GPT_4 = ModelType + var GPT_4O = ModelType + var GPT_4O_MINI = ModelType + var GPT_4_32K = ModelType + var GPT_4_TURBO = ModelType + var TEXT_ADA_001 = ModelType + var TEXT_BABBAGE_001 = ModelType + var TEXT_CURIE_001 = ModelType + var TEXT_DAVINCI_001 = ModelType + var TEXT_DAVINCI_002 = ModelType + var TEXT_DAVINCI_003 = ModelType + var TEXT_DAVINCI_EDIT_001 = ModelType + var TEXT_EMBEDDING_3_LARGE = ModelType + var TEXT_EMBEDDING_3_SMALL = ModelType + var TEXT_EMBEDDING_ADA_002 = ModelType + var TEXT_SEARCH_ADA_DOC_001 = ModelType + var TEXT_SEARCH_BABBAGE_DOC_001 = ModelType + var TEXT_SEARCH_CURIE_DOC_001 = ModelType + var TEXT_SEARCH_DAVINCI_DOC_001 = ModelType + var TEXT_SIMILARITY_ADA_001 = ModelType + var TEXT_SIMILARITY_BABBAGE_001 = ModelType + var TEXT_SIMILARITY_CURIE_001 = ModelType + var TEXT_SIMILARITY_DAVINCI_001 = ModelType + var VERY_LARGE_TOKENIZER_BYTE_THRESHOLD_KEY = "VERY_LARGE_TOKENIZER_BYTE_THRESHOLD" + type Encoding interface + CountTokens func(text string) int + CountTokensOrdinary func(text string) int + Decode func(tokens []int) string + DecodeBytes func(tokens []int) []byte + Encode func(text string, maxTokens int) *EncodingResult + EncodeOrdinary func(text string, maxTokens int) *EncodingResult + EncodeOrdinaryToIntArray func(text string) []int + EncodeToIntArray func(text string) []int + GetName func() string + type EncodingRegistry interface + GetEncoding func(encodingName string) (Encoding, error) + GetEncodingByType func(encodingType EncodingType) (Encoding, error) + GetEncodingForModel func(modelName string) (Encoding, error) + GetEncodingForModelType func(modelType ModelType) (Encoding, error) + RegisterCustomEncoding func(encoding Encoding) (EncodingRegistry, error) + RegisterGptBytePairEncoding func(parameters *GptBytePairEncodingParams) (EncodingRegistry, error) + type EncodingResult struct + func NewEncodingResult(tokens []int, truncated bool, lastProcessedCharacterIndex ...int) *EncodingResult + func (er *EncodingResult) GetLastProcessedCharacterIndex() int + func (er *EncodingResult) GetTokens() []int + func (er *EncodingResult) IsTruncated() bool + func (er *EncodingResult) ToString() string + type EncodingType string + const CL100K_BASE + const O200K_BASE + const P50K_BASE + const P50K_EDIT + const R50K_BASE + func EncodingTypeFromName(name string) (EncodingType, bool) + func EncodingTypeValues() []EncodingType + func (e EncodingType) GetName() string + type GptBytePairEncodingParams struct + func NewGptBytePairEncodingParams(name string, pattern *regexp.Regexp, encoder map[string]int, ...) *GptBytePairEncodingParams + func (g *GptBytePairEncodingParams) GetEncoder() map[string]int + func (g *GptBytePairEncodingParams) GetName() string + func (g *GptBytePairEncodingParams) GetPattern() *regexp.Regexp + func (g *GptBytePairEncodingParams) GetSpecialTokensEncoder() map[string]int + type ModelType struct + func ModelTypeFromName(name string) (*ModelType, bool) + func ModelTypeValues() []ModelType + func (m *ModelType) GetEncodingType() EncodingType + func (m *ModelType) GetMaxContextLength() int + func (m *ModelType) GetName() string
Go to main page