tokenizer

README

Tokenizer

Fork of github.com/awee-ai/go-tokenizer with (currently WIP) incorrect Ollama model tokenizer mappings.

Maybe something will get done here.

https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained

Anthropic

python3 internal/anthropic/anthropic.py internal/anthropic/tokenizer.json internal/anthropic/anthropic.tiktoken

go run ./internal/cmd/vocab.go -encoding anthropic

Documentation

Index

• Variables
• func Count(model Model, input string) (int, error)
• type Codec
  • func ForModel(model Model) (Codec, error)
  • func Get(encoding Encoding) (Codec, error)
• type Encoding
• type Model

Variables

var (
	ErrModelNotSupported    = errors.New("model not supported")
	ErrEncodingNotSupported = errors.New("encoding not supported")
)

var Ratios = map[string]float64{
	"claude-": 1.166,
}

Functions

func Count

func Count(model Model, input string) (int, error)

Types

type Codec

type Codec interface {
	GetName() string
	Count(string) (int, error)
	Encode(string) ([]uint, []string, error)
	Decode([]uint) (string, error)
}

func ForModel

func ForModel(model Model) (Codec, error)

ForModel returns a new instance of a Codec implementation based on the specified OpenAI model. If the specified model is not supported, an error is returned.

func Get

func Get(encoding Encoding) (Codec, error)

Get returns a new instance of a Codec implementation based on the specified encoding format. The returned Codec instance can be used to encode (tokenize) and decode (reassemble) text. If the specified encoding is not supported, an error is returned.

type Encoding

type Encoding string

const (
	R50kBase        Encoding = "r50k_base"   // OpenAI GPT-2 base tokenizer (same as above)
	P50kBase        Encoding = "p50k_base"   // Codex tokenizer variant (GPT-style BPE, 50k)
	P50kEdit        Encoding = "p50k_edit"   // Used by OpenAI's edit models
	Cl100kBase      Encoding = "cl100k_base" // GPT-4/GPT-3.5 Turbo tokenizer (100k BPE)
	O200kBase       Encoding = "o200k_base"  // OpenAI 200k tokenizer (e.g. GPT-4o, o1, o3)
	OllamaLlamaBase Encoding = "llama"       // LLaMA3 tokenizer (BPE, 200k vocab, used by LLama3+ models)
	AnthropicBase   Encoding = "cl100k_base" // Anthropic tokenizer (Claude family, 100k vocab)
	GeminiBase      Encoding = "cl100k_base" // Gemini tokenizer (200k vocab, tiktoken-compatible)

)

type Model

type Model string

const (
	O1                       Model = "o1"
	O1Preview                Model = "o1-preview"
	O1Mini                   Model = "o1-mini"
	O3                       Model = "o3"
	O3Mini                   Model = "o3-mini"
	O4Mini                   Model = "o4-mini"
	GPT41                    Model = "gpt-4.1"
	GPT4o                    Model = "gpt-4o"
	GPT4                     Model = "gpt-4"
	GPT35Turbo               Model = "gpt-3.5-turbo"
	GPT35                    Model = "gpt-3.5"
	TextEmbeddingAda002      Model = "text-embedding-ada-002"
	TextDavinci003           Model = "text-davinci-003"
	TextDavinci002           Model = "text-davinci-002"
	CodeDavinci002           Model = "code-davinci-002"
	CodeDavinci001           Model = "code-davinci-001"
	CodeCushman002           Model = "code-cushman-002"
	CodeCushman001           Model = "code-cushman-001"
	DavinciCodex             Model = "davinci-codex"
	CushmanCodex             Model = "cushman-codex"
	TextDavinci001           Model = "text-davinci-001"
	TextCurie001             Model = "text-curie-001"
	TextBabbage001           Model = "text-babbage-001"
	TextAda001               Model = "text-ada-001"
	Davinci                  Model = "davinci"
	Curie                    Model = "curie"
	Babbage                  Model = "babbage"
	Ada                      Model = "ada"
	TextSimilarityDavinci001 Model = "text-similarity-davinci-001"
	TextSimilarityCurie001   Model = "text-similarity-curie-001"
	TextSimilarityBabbage001 Model = "text-similarity-babbage-001"
	TextSimilarityAda001     Model = "text-similarity-ada-001"
	TextSearchDavinciDoc001  Model = "text-search-davinci-doc-001"
	TextSearchCurieDoc001    Model = "text-search-curie-doc-001"
	TextSearchAdaDoc001      Model = "text-search-ada-doc-001"
	TextSearchBabbageDoc001  Model = "text-search-babbage-doc-001"
	CodeSearchBabbageCode001 Model = "code-search-babbage-code-001"
	CodeSearchAdaCode001     Model = "code-search-ada-code-001"
	TextDavinciEdit001       Model = "text-davinci-edit-001"
	CodeDavinciEdit001       Model = "code-davinci-edit-001"
)