batch

Documentation

Index

• Constants
• func GetTokensCount(model string, input string, tke *tiktoken.Tiktoken) int
• type Batch
  • func NewBatchVectorizer[T dto.Embedding](client BatchClient[T], maxBatchTime time.Duration, settings Settings, ...) *Batch[T]
  • func (b *Batch[T]) SubmitBatchAndWait(ctx context.Context, cfg moduletools.ClassConfig, skipObject []bool, ...) ([]T, map[int]error)
• type BatchClient
• type BatchJob
• type BatchSimple
  • func NewBatchSimple[T dto.Embedding](logger logrus.FieldLogger, rpm int) *BatchSimple[T]
  • func (b *BatchSimple[T]) VectorizeBatch(ctx context.Context, objs []*models.Object, skipObject []bool, ...) ([]T, []models.AdditionalProperties, map[int]error)
  • func (b *BatchSimple[T]) VectorizeBatchObjects(ctx context.Context, objs []*models.Object, skipObject []bool, ...) ([]T, []models.AdditionalProperties, map[int]error)
• type EncoderCache
  • func NewEncoderCache() *EncoderCache
  • func (e *EncoderCache) Get(model string) (*tiktoken.Tiktoken, bool)
  • func (e *EncoderCache) Set(model string, tk *tiktoken.Tiktoken)
• type Settings
• type TokenizerFuncType
  • func ReturnBatchTokenizer(multiplier float32, moduleName string, lowerCaseInput bool) TokenizerFuncType
  • func ReturnBatchTokenizerWithAltNames(multiplier float32, moduleName string, altNames []string, lowerCaseInput bool) TokenizerFuncType

Constants

const BatchChannelSize = 100

Functions

func GetTokensCount

func GetTokensCount(model string, input string, tke *tiktoken.Tiktoken) int

Types

type Batch

type Batch[T dto.Embedding] struct {
	Label string
	// contains filtered or unexported fields
}

func NewBatchVectorizer

func NewBatchVectorizer[T dto.Embedding](client BatchClient[T], maxBatchTime time.Duration, settings Settings, logger logrus.FieldLogger, label string) *Batch[T]

func (*Batch[T]) SubmitBatchAndWait

func (b *Batch[T]) SubmitBatchAndWait(ctx context.Context, cfg moduletools.ClassConfig, skipObject []bool, tokenCounts []int, texts []string) ([]T, map[int]error)

type BatchClient

type BatchClient[T dto.Embedding] interface {
	Vectorize(ctx context.Context, input []string,
		config moduletools.ClassConfig) (*modulecomponents.VectorizationResult[T], *modulecomponents.RateLimits, int, error)
	GetVectorizerRateLimit(ctx context.Context, config moduletools.ClassConfig) *modulecomponents.RateLimits
	GetApiKeyHash(ctx context.Context, config moduletools.ClassConfig) [32]byte
}

type BatchJob

type BatchJob[T dto.Embedding] struct {
	// contains filtered or unexported fields
}

type BatchSimple

type BatchSimple[T dto.Embedding] struct {
	// contains filtered or unexported fields
}

BatchSimple exposes the per-object (VectorizeBatch) and per-batch (VectorizeBatchObjects) vectorization helpers used by modules that do not need the full token-aware Batch[T] pipeline.

When a positive RPM is supplied to NewBatchSimple, requests are paced by a token-bucket limiter. One limiter token is reserved per input embedding (NOT per HTTP request) because some providers — e.g. Google Gemini — count each embedding as a separate request against the RPM quota even when multiple inputs are batched into a single HTTP call.

func NewBatchSimple

func NewBatchSimple[T dto.Embedding](logger logrus.FieldLogger, rpm int) *BatchSimple[T]

NewBatchSimple returns a BatchSimple. When rpm > 0 requests are limited to approximately rpm embeddings per minute with a burst of max(rpm/60, 1). When rpm <= 0 no rate limiting is applied.

func (*BatchSimple[T]) VectorizeBatch

func (b *BatchSimple[T]) VectorizeBatch(
	ctx context.Context,
	objs []*models.Object,
	skipObject []bool,
	cfg moduletools.ClassConfig,
	vectorize objectVectorizer[T],
) ([]T, []models.AdditionalProperties, map[int]error)

func (*BatchSimple[T]) VectorizeBatchObjects

func (b *BatchSimple[T]) VectorizeBatchObjects(
	ctx context.Context,
	objs []*models.Object,
	skipObject []bool,
	cfg moduletools.ClassConfig,
	vectorize batchObjectsVectorizer[T],
	batchSize int,
) ([]T, []models.AdditionalProperties, map[int]error)

type EncoderCache

type EncoderCache struct {
	// contains filtered or unexported fields
}

func NewEncoderCache

func NewEncoderCache() *EncoderCache

func (*EncoderCache) Get

func (e *EncoderCache) Get(model string) (*tiktoken.Tiktoken, bool)

func (*EncoderCache) Set

func (e *EncoderCache) Set(model string, tk *tiktoken.Tiktoken)

type Settings

type Settings struct {
	TokenMultiplier    float32
	MaxTimePerBatch    float64
	MaxObjectsPerBatch int
	MaxTokensPerBatch  func(cfg moduletools.ClassConfig) int
	HasTokenLimit      bool
	ReturnsRateLimit   bool
}

type TokenizerFuncType

type TokenizerFuncType func(ctx context.Context, objects []*models.Object, skipObject []bool, cfg moduletools.ClassConfig, objectVectorizer *objectsvectorizer.ObjectVectorizer, encoderCache *EncoderCache) ([]string, []int, []bool, bool, error)

func ReturnBatchTokenizer

func ReturnBatchTokenizer(multiplier float32, moduleName string, lowerCaseInput bool) TokenizerFuncType

func ReturnBatchTokenizerWithAltNames

func ReturnBatchTokenizerWithAltNames(multiplier float32, moduleName string, altNames []string, lowerCaseInput bool) TokenizerFuncType