mod

Documentation

Index

• Variables
• type Encoding
• type EncodingRegistry
• type EncodingResult
  • func NewEncodingResult(tokens []int, truncated bool, lastProcessedCharacterIndex ...int) *EncodingResult
  • func (er *EncodingResult) GetLastProcessedCharacterIndex() int
  • func (er *EncodingResult) GetTokens() []int
  • func (er *EncodingResult) IsTruncated() bool
  • func (er *EncodingResult) ToString() string
• type EncodingType
  • func EncodingTypeFromName(name string) (EncodingType, bool)
  • func EncodingTypeValues() []EncodingType
  • func (e EncodingType) GetName() string
• type GptBytePairEncodingParams
  • func NewGptBytePairEncodingParams(name string, pattern *regexp.Regexp, encoder map[string]int, ...) *GptBytePairEncodingParams
  • func (g *GptBytePairEncodingParams) GetEncoder() map[string]int
  • func (g *GptBytePairEncodingParams) GetName() string
  • func (g *GptBytePairEncodingParams) GetPattern() *regexp.Regexp
  • func (g *GptBytePairEncodingParams) GetSpecialTokensEncoder() map[string]int
• type ModelType
  • func ModelTypeFromName(name string) (*ModelType, bool)
  • func ModelTypeValues() []ModelType
  • func (m *ModelType) GetEncodingType() EncodingType
  • func (m *ModelType) GetMaxContextLength() int
  • func (m *ModelType) GetName() string

Variables

var (
	// Chat models
	GPT_4             = ModelType{"gpt-4", CL100K_BASE, 8192}
	GPT_4O            = ModelType{"gpt-4o", O200K_BASE, 128000}
	GPT_4O_MINI       = ModelType{"gpt-4o-mini", O200K_BASE, 128000}
	GPT_4_32K         = ModelType{"gpt-4-32k", CL100K_BASE, 32768}
	GPT_4_TURBO       = ModelType{"gpt-4-turbo", CL100K_BASE, 128000}
	GPT_3_5_TURBO     = ModelType{"gpt-3.5-turbo", CL100K_BASE, 16385}
	GPT_3_5_TURBO_16K = ModelType{"gpt-3.5-turbo-16k", CL100K_BASE, 16385}

	// Text models
	TEXT_DAVINCI_003 = ModelType{"text-davinci-003", P50K_BASE, 4097}
	TEXT_DAVINCI_002 = ModelType{"text-davinci-002", P50K_BASE, 4097}
	TEXT_DAVINCI_001 = ModelType{"text-davinci-001", R50K_BASE, 2049}
	TEXT_CURIE_001   = ModelType{"text-curie-001", R50K_BASE, 2049}
	TEXT_BABBAGE_001 = ModelType{"text-babbage-001", R50K_BASE, 2049}
	TEXT_ADA_001     = ModelType{"text-ada-001", R50K_BASE, 2049}
	DAVINCI          = ModelType{"davinci", R50K_BASE, 2049}
	CURIE            = ModelType{"curie", R50K_BASE, 2049}
	BABBAGE          = ModelType{"babbage", R50K_BASE, 2049}
	ADA              = ModelType{"ada", R50K_BASE, 2049}

	// Code models
	CODE_DAVINCI_002 = ModelType{"code-davinci-002", P50K_BASE, 8001}
	CODE_DAVINCI_001 = ModelType{"code-davinci-001", P50K_BASE, 8001}
	CODE_CUSHMAN_002 = ModelType{"code-cushman-002", P50K_BASE, 2048}
	CODE_CUSHMAN_001 = ModelType{"code-cushman-001", P50K_BASE, 2048}
	DAVINCI_CODEX    = ModelType{"davinci-codex", P50K_BASE, 4096}
	CUSHMAN_CODEX    = ModelType{"cushman-codex", P50K_BASE, 2048}

	// Edit models
	TEXT_DAVINCI_EDIT_001 = ModelType{"text-davinci-edit-001", P50K_EDIT, 3000}
	CODE_DAVINCI_EDIT_001 = ModelType{"code-davinci-edit-001", P50K_EDIT, 3000}

	// Embeddings
	TEXT_EMBEDDING_ADA_002 = ModelType{"text-embedding-ada-002", CL100K_BASE, 8191}
	TEXT_EMBEDDING_3_SMALL = ModelType{"text-embedding-3-small", CL100K_BASE, 8191}
	TEXT_EMBEDDING_3_LARGE = ModelType{"text-embedding-3-large", CL100K_BASE, 8191}

	// Old embeddings
	TEXT_SIMILARITY_DAVINCI_001  = ModelType{"text-similarity-davinci-001", R50K_BASE, 2046}
	TEXT_SIMILARITY_CURIE_001    = ModelType{"text-similarity-curie-001", R50K_BASE, 2046}
	TEXT_SIMILARITY_BABBAGE_001  = ModelType{"text-similarity-babbage-001", R50K_BASE, 2046}
	TEXT_SIMILARITY_ADA_001      = ModelType{"text-similarity-ada-001", R50K_BASE, 2046}
	TEXT_SEARCH_DAVINCI_DOC_001  = ModelType{"text-search-davinci-doc-001", R50K_BASE, 2046}
	TEXT_SEARCH_CURIE_DOC_001    = ModelType{"text-search-curie-doc-001", R50K_BASE, 2046}
	TEXT_SEARCH_BABBAGE_DOC_001  = ModelType{"text-search-babbage-doc-001", R50K_BASE, 2046}
	TEXT_SEARCH_ADA_DOC_001      = ModelType{"text-search-ada-doc-001", R50K_BASE, 2046}
	CODE_SEARCH_BABBAGE_CODE_001 = ModelType{"code-search-babbage-code-001", R50K_BASE, 2046}
	CODE_SEARCH_ADA_CODE_001     = ModelType{"code-search-ada-code-001", R50K_BASE, 2046}
)

var VERY_LARGE_TOKENIZER_BYTE_THRESHOLD_KEY = "VERY_LARGE_TOKENIZER_BYTE_THRESHOLD"

Types

type Encoding

type Encoding interface {
	EncodeToIntArray(text string) []int
	Encode(text string, maxTokens int) *EncodingResult
	EncodeOrdinaryToIntArray(text string) []int
	EncodeOrdinary(text string, maxTokens int) *EncodingResult
	CountTokens(text string) int
	CountTokensOrdinary(text string) int
	Decode(tokens []int) string
	DecodeBytes(tokens []int) []byte
	GetName() string
}

type EncodingRegistry

type EncodingRegistry interface {
	GetEncoding(encodingName string) (Encoding, error)
	GetEncodingByType(encodingType EncodingType) (Encoding, error)
	GetEncodingForModel(modelName string) (Encoding, error)
	GetEncodingForModelType(modelType ModelType) (Encoding, error)
	RegisterGptBytePairEncoding(parameters *GptBytePairEncodingParams) (EncodingRegistry, error)
	RegisterCustomEncoding(encoding Encoding) (EncodingRegistry, error)
}

type EncodingResult

type EncodingResult struct {
	// contains filtered or unexported fields
}

The result of encoding operation.

func NewEncodingResult

func NewEncodingResult(tokens []int, truncated bool, lastProcessedCharacterIndex ...int) *EncodingResult

func (*EncodingResult) GetLastProcessedCharacterIndex

func (er *EncodingResult) GetLastProcessedCharacterIndex() int

func (*EncodingResult) GetTokens

func (er *EncodingResult) GetTokens() []int

func (*EncodingResult) IsTruncated

func (er *EncodingResult) IsTruncated() bool

func (*EncodingResult) ToString

func (er *EncodingResult) ToString() string

type EncodingType

type EncodingType string

const (
	R50K_BASE   EncodingType = "r50k_base"
	P50K_BASE   EncodingType = "p50k_base"
	P50K_EDIT   EncodingType = "p50k_edit"
	CL100K_BASE EncodingType = "cl100k_base"
	O200K_BASE  EncodingType = "o200k_base"
)

func EncodingTypeFromName

func EncodingTypeFromName(name string) (EncodingType, bool)

func EncodingTypeValues

func EncodingTypeValues() []EncodingType

func (EncodingType) GetName

func (e EncodingType) GetName() string

type GptBytePairEncodingParams

type GptBytePairEncodingParams struct {
	// contains filtered or unexported fields
}

func NewGptBytePairEncodingParams

func NewGptBytePairEncodingParams(
	name string,
	pattern *regexp.Regexp,
	encoder map[string]int,
	specialTokensEncoder map[string]int,
) *GptBytePairEncodingParams

func (*GptBytePairEncodingParams) GetEncoder

func (g *GptBytePairEncodingParams) GetEncoder() map[string]int

func (*GptBytePairEncodingParams) GetName

func (g *GptBytePairEncodingParams) GetName() string

func (*GptBytePairEncodingParams) GetPattern

func (g *GptBytePairEncodingParams) GetPattern() *regexp.Regexp

func (*GptBytePairEncodingParams) GetSpecialTokensEncoder

func (g *GptBytePairEncodingParams) GetSpecialTokensEncoder() map[string]int

type ModelType

type ModelType struct {
	// contains filtered or unexported fields
}

func ModelTypeFromName

func ModelTypeFromName(name string) (*ModelType, bool)

func ModelTypeValues

func ModelTypeValues() []ModelType

func (*ModelType) GetEncodingType

func (m *ModelType) GetEncodingType() EncodingType

func (*ModelType) GetMaxContextLength

func (m *ModelType) GetMaxContextLength() int

func (*ModelType) GetName

func (m *ModelType) GetName() string