tokenizer

package

Go to main page

Versions in this module

v0

v0.6.0

Aug 5, 2026

v0.5.1

Jul 30, 2026

Changes in this version

type BPETokenizerWrapper

+ func (t *BPETokenizerWrapper) Contract() ContractIdentity

type ClaudeApproximator

+ func (c *ClaudeApproximator) Contract() ContractIdentity

+ type ContractIdentity struct

+ Encoding string

+ Implementation string

+ Method string

+ NormalizationPolicy string

+ SpecialTokenPolicy string

+ VocabularyDigest [sha256.Size]byte

+ func ContractOf(tokenizer Tokenizer) (ContractIdentity, bool)

+ func (identity ContractIdentity) Valid() bool

+ type ContractIdentityProvider interface

+ Contract func() ContractIdentity

+ type CountFilesOptions struct

+ All bool

+ Model string

+ OnProgress ProgressFunc

type Counter

+ func (c *Counter) CountFilesWithCache(ctx context.Context, root string, files []string, model string, all bool, ...) (*CountResult, error)

+ func (c *Counter) CountFilesWithCacheOptions(ctx context.Context, root string, files []string, opts CountFilesOptions, ...) (*CountResult, error)

+ func (c *Counter) CountFilesWithOptions(ctx context.Context, files []string, opts CountFilesOptions) (*CountResult, error)

type CounterOptions

+ Stats *Stats

type GeminiApproximator

+ func (g *GeminiApproximator) Contract() ContractIdentity

+ type ProgressFunc func(ProgressUpdate)

+ type ProgressUpdate struct

+ Bytes int64

+ Characters int

+ FilesDone int

+ FilesTotal int

+ LastPath string

+ Lines int

+ MethodTokens []int

+ Words int

type SPMTokenizerWrapper

+ func (t *SPMTokenizerWrapper) Contract() ContractIdentity

+ type Stats struct

+ func NewStats() *Stats

+ func (s *Stats) ObserveMemory()

+ func (s *Stats) RecordAggregationDuration(duration time.Duration)

+ func (s *Stats) RecordBinarySniffBytes(bytes int64)

+ func (s *Stats) RecordBinarySniffOpen()

+ func (s *Stats) RecordCacheBytesReused(bytes int64)

+ func (s *Stats) RecordCacheHit(reason string, methods int)

+ func (s *Stats) RecordCacheMiss(reason string)

+ func (s *Stats) RecordCachePartialHit(reason string, reusableMethods, missingMethods int)

+ func (s *Stats) RecordCacheWarning()

+ func (s *Stats) RecordEligibleFile()

+ func (s *Stats) RecordEntryVisited()

+ func (s *Stats) RecordFullFileBytes(bytes int64)

+ func (s *Stats) RecordFullFileOpen()

+ func (s *Stats) RecordPersistenceReadyDuration(duration time.Duration)

+ func (s *Stats) RecordTokenizationDuration(duration time.Duration)

+ func (s *Stats) RecordTokenizedFile(method string)

+ func (s *Stats) RecordValidationReadDuration(duration time.Duration)

+ func (s *Stats) RecordWalkDuration(duration time.Duration)

+ func (s *Stats) Snapshot() StatsSnapshot

+ type StatsSnapshot struct

+ AggregationDuration time.Duration

+ BinarySniffBytes int64

+ BinarySniffOpens int64

+ CacheBytesReused int64

+ CacheHits int64

+ CacheMethodsAvoided int64

+ CacheMisses int64

+ CachePartialHits int64

+ CacheReasons map[string]int64

+ CacheWarnings int64

+ EligibleFiles int64

+ EntriesVisited int64

+ FilesTokenizedByMethod map[string]int64

+ FullFileBytes int64

+ FullFileOpens int64

+ PeakHeapAllocBytes uint64

+ PersistenceReadyDuration time.Duration

+ TokenizationDuration time.Duration

+ ValidationReadDuration time.Duration

+ WalkDuration time.Duration

v0.5.0

Jul 12, 2026

Changes in this version

+ const DefaultCharsPerToken

+ const DefaultWordsPerToken

+ const EncodingCL100kBase

+ const EncodingClaudeApprox

+ const EncodingGeminiApprox

+ const EncodingO200kBase

+ const EncodingSPM

+ const NameClaudeApprox

type Counter

+ func (c *Counter) CountFiles(ctx context.Context, files []string, model string, all bool) (*CountResult, error)

type ModelMetadata

+ func LookupModel(modelName string) *ModelMetadata

type Provider

+ func ProviderForModel(modelName string) Provider

v0.4.0

Jul 9, 2026

Changes in this version

+ type GeminiApproximator struct

+ func (g *GeminiApproximator) CountTokens(text string) (int, error)

+ func (g *GeminiApproximator) DisplayName() string

+ func (g *GeminiApproximator) IsExact() bool

+ func (g *GeminiApproximator) Name() string

type Tokenizer

+ func NewGeminiApproximator() Tokenizer

v0.3.0

Feb 24, 2026

Changes in this version

+ var ErrBinaryFile = errors.New("file is binary")

+ var ErrEncodingNotFound = errors.New("encoding not found")

+ var ErrModelNotFound = errors.New("model not found")

+ var ErrVocabFileRequired = errors.New("vocab file path is required")

+ func IsOpenSourceModel(modelName string) bool

+ func ListModels() []string

+ func ModelsByEncoding() map[string][]string

+ type BPETokenizerWrapper struct

+ func (t *BPETokenizerWrapper) CountTokens(text string) (int, error)

+ func (t *BPETokenizerWrapper) DisplayName() string

+ func (t *BPETokenizerWrapper) IsExact() bool

+ func (t *BPETokenizerWrapper) Name() string

+ type ClaudeApproximator struct

+ func (c *ClaudeApproximator) CountTokens(text string) (int, error)

+ func (c *ClaudeApproximator) DisplayName() string

+ func (c *ClaudeApproximator) IsExact() bool

+ func (c *ClaudeApproximator) Name() string

+ type CostEstimate struct

+ Cost float64

+ Model string

+ RatePer1M float64

+ Tokens int

+ func CalculateCosts(methods []MethodResult) []CostEstimate

+ type CountResult struct

+ Characters int

+ Costs []CostEstimate

+ FileCount int

+ FilePath string

+ FileSize int

+ IsDirectory bool

+ Lines int

+ Methods []MethodResult

+ Words int

+ type Counter struct

+ func NewCounter(opts CounterOptions) (*Counter, error)

+ func (c *Counter) Count(ctx context.Context, text string, model string, all bool) (*CountResult, error)

+ func (c *Counter) CountDirectory(ctx context.Context, path string, model string, all bool) (*CountResult, error)

+ func (c *Counter) CountFile(ctx context.Context, path string, model string, all bool) (*CountResult, error)

+ type CounterOptions struct

+ CharsPerToken float64

+ Provider Provider

+ VocabFile string

+ WordsPerToken float64

+ type MethodResult struct

+ ContextWindow int

+ DisplayName string

+ IsExact bool

+ Name string

+ Tokens int

+ type ModelMetadata struct

+ ContextWindow int

+ Encoding string

+ InputPricePer1M float64

+ Name string

+ OutputPricePer1M float64

+ Provider Provider

+ func GetModelMetadata(modelName string) *ModelMetadata

+ func GetPricingForModel(model string) *ModelMetadata

+ func ListModelsByProvider(provider Provider) []ModelMetadata

+ type Provider string

+ const ProviderAlibaba

+ const ProviderAnthropic

+ const ProviderDeepSeek

+ const ProviderGoogle

+ const ProviderMeta

+ const ProviderMicrosoft

+ const ProviderOpenAI

+ func GetProviderForModel(modelName string) Provider

+ type SPMTokenizerWrapper struct

+ func (t *SPMTokenizerWrapper) CountTokens(text string) (int, error)

+ func (t *SPMTokenizerWrapper) DisplayName() string

+ func (t *SPMTokenizerWrapper) IsExact() bool

+ func (t *SPMTokenizerWrapper) Name() string

+ type Tokenizer interface

+ CountTokens func(text string) (int, error)

+ DisplayName func() string

+ IsExact func() bool

+ Name func() string

+ func NewBPETokenizer(model string) (Tokenizer, error)

+ func NewBPETokenizerByEncoding(encodingName string) (Tokenizer, error)

+ func NewClaudeApproximator() Tokenizer

+ func NewSPMTokenizer(modelPath string) (Tokenizer, error)

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL