text

Documentation

Index

• type Parser
  • func NewParser() *Parser
  • func (p *Parser) Parse(ctx context.Context, r io.Reader) ([]core.Chunk, error)
  • func (p *Parser) ParseWithCallback(ctx context.Context, r io.Reader, callback func(core.Chunk) error) error
  • func (p *Parser) SupportedFormats() []string
• type SemanticParser
  • func NewSemanticParser(embedder embedding.Provider) *SemanticParser
  • func (p *SemanticParser) Parse(ctx context.Context, r io.Reader) ([]core.Chunk, error)
  • func (p *SemanticParser) SupportedFormats() []string
• type StreamingParser
  • func NewStreamingParser() *StreamingParser
  • func (p *StreamingParser) Parse(ctx context.Context, r io.Reader) ([]core.Chunk, error)
  • func (p *StreamingParser) SupportedFormats() []string

Types

type Parser

type Parser struct {
	// contains filtered or unexported fields
}

Parser implements a simple text parser

func NewParser

func NewParser() *Parser

NewParser creates a new text parser

func (*Parser) Parse

func (p *Parser) Parse(ctx context.Context, r io.Reader) ([]core.Chunk, error)

Parse parses text into chunks

func (*Parser) ParseWithCallback

func (p *Parser) ParseWithCallback(ctx context.Context, r io.Reader, callback func(core.Chunk) error) error

ParseWithCallback parses text and calls the callback for each chunk

func (*Parser) SupportedFormats

func (p *Parser) SupportedFormats() []string

SupportedFormats returns supported formats

type SemanticParser

type SemanticParser struct {
	// contains filtered or unexported fields
}

SemanticParser implements semantic chunking based on sentence boundaries and semantic similarity

func NewSemanticParser

func NewSemanticParser(embedder embedding.Provider) *SemanticParser

NewSemanticParser creates a new semantic parser

func (*SemanticParser) Parse

func (p *SemanticParser) Parse(ctx context.Context, r io.Reader) ([]core.Chunk, error)

Parse parses text into semantically meaningful chunks

func (*SemanticParser) SupportedFormats

func (p *SemanticParser) SupportedFormats() []string

SupportedFormats returns supported formats

type StreamingParser

type StreamingParser struct {
	// contains filtered or unexported fields
}

func NewStreamingParser

func NewStreamingParser() *StreamingParser

NewStreamingParser creates a new streaming text parser

func (*StreamingParser) Parse

func (p *StreamingParser) Parse(ctx context.Context, r io.Reader) ([]core.Chunk, error)

Parse parses text into chunks using streaming processing

func (*StreamingParser) SupportedFormats

func (p *StreamingParser) SupportedFormats() []string

SupportedFormats returns supported formats