reembedding

Documentation

Index

• type CollectionReader
  • func NewCollectionReader(client vectordb.VectorDBClient, collectionName string, batchSize int) (*CollectionReader, error)
  • func (r *CollectionReader) HasMore() bool
  • func (r *CollectionReader) Progress() (int, int64)
  • func (r *CollectionReader) ReadBatch(ctx context.Context) ([]*vectordb.Document, error)
  • func (r *CollectionReader) Reset()
• type EmbeddingPipeline
  • func NewEmbeddingPipeline(modelName string) (*EmbeddingPipeline, error)
  • func (p *EmbeddingPipeline) GetDimensions() int
  • func (p *EmbeddingPipeline) GetModelName() string
  • func (p *EmbeddingPipeline) GetProvider() string
  • func (p *EmbeddingPipeline) IsSupported() bool
  • func (p *EmbeddingPipeline) ProcessBatch(ctx context.Context, docs []*vectordb.Document) error
• type ProgressTracker
  • func NewProgressTracker(total int64) *ProgressTracker
  • func (p *ProgressTracker) Display()
  • func (p *ProgressTracker) GetETA() time.Duration
  • func (p *ProgressTracker) GetPercentage() float64
  • func (p *ProgressTracker) GetSpeed() float64
  • func (p *ProgressTracker) GetStatusMessage() string
  • func (p *ProgressTracker) IsComplete() bool
  • func (p *ProgressTracker) Update(count int)

Types

type CollectionReader

type CollectionReader struct {
	// contains filtered or unexported fields
}

CollectionReader reads documents from a source collection in paginated batches

func NewCollectionReader

func NewCollectionReader(client vectordb.VectorDBClient, collectionName string, batchSize int) (*CollectionReader, error)

NewCollectionReader creates a new CollectionReader for the given collection

func (*CollectionReader) HasMore

func (r *CollectionReader) HasMore() bool

HasMore returns true if there are more documents to read

func (*CollectionReader) Progress

func (r *CollectionReader) Progress() (int, int64)

Progress returns the current progress (documents read, total documents)

func (*CollectionReader) ReadBatch

func (r *CollectionReader) ReadBatch(ctx context.Context) ([]*vectordb.Document, error)

ReadBatch reads the next batch of documents from the collection

func (*CollectionReader) Reset

func (r *CollectionReader) Reset()

Reset resets the reader to the beginning of the collection

type EmbeddingPipeline

type EmbeddingPipeline struct {
	// contains filtered or unexported fields
}

EmbeddingPipeline handles re-generating embeddings for documents

func NewEmbeddingPipeline

func NewEmbeddingPipeline(modelName string) (*EmbeddingPipeline, error)

NewEmbeddingPipeline creates a new embedding pipeline with auto-detected dimensions

func (*EmbeddingPipeline) GetDimensions

func (p *EmbeddingPipeline) GetDimensions() int

GetDimensions returns the vector dimensions for this pipeline

func (*EmbeddingPipeline) GetModelName

func (p *EmbeddingPipeline) GetModelName() string

GetModelName returns the embedding model name

func (*EmbeddingPipeline) GetProvider

func (p *EmbeddingPipeline) GetProvider() string

GetProvider returns the embedding provider

func (*EmbeddingPipeline) IsSupported

func (p *EmbeddingPipeline) IsSupported() bool

IsSupported returns true if the model is supported for re-embedding

func (*EmbeddingPipeline) ProcessBatch

func (p *EmbeddingPipeline) ProcessBatch(ctx context.Context, docs []*vectordb.Document) error

ProcessBatch generates new embeddings for a batch of documents

type ProgressTracker

type ProgressTracker struct {
	// contains filtered or unexported fields
}

ProgressTracker tracks and displays progress for re-embedding operations

func NewProgressTracker

func NewProgressTracker(total int64) *ProgressTracker

NewProgressTracker creates a new progress tracker

func (*ProgressTracker) Display

func (p *ProgressTracker) Display()

Display shows the current progress with a progress bar and ETA

func (*ProgressTracker) GetETA

func (p *ProgressTracker) GetETA() time.Duration

GetETA returns the estimated time remaining

func (*ProgressTracker) GetPercentage

func (p *ProgressTracker) GetPercentage() float64

GetPercentage returns the completion percentage

func (*ProgressTracker) GetSpeed

func (p *ProgressTracker) GetSpeed() float64

GetSpeed returns the current processing speed in documents per minute

func (*ProgressTracker) GetStatusMessage

func (p *ProgressTracker) GetStatusMessage() string

GetStatusMessage returns formatted progress message without printing

func (*ProgressTracker) IsComplete

func (p *ProgressTracker) IsComplete() bool

IsComplete returns true if all documents have been processed

func (*ProgressTracker) Update

func (p *ProgressTracker) Update(count int)

Update increments the progress counter