chunker

Documentation

Index

• type Chunk
• type Chunker
  • func New(chunkSize, chunkOverlap int) *Chunker
  • func (c *Chunker) Split(text string) []Chunk

Types

type Chunk

type Chunk struct {
	Index   int
	Content string
	Tokens  int
}

type Chunker

type Chunker struct {
	// contains filtered or unexported fields
}

func New

func New(chunkSize, chunkOverlap int) *Chunker

func (*Chunker) Split

func (c *Chunker) Split(text string) []Chunk