html

Documentation

Index

• type Content
• type Loader
  • func NewLoader(opts ...Option) *Loader
  • func (l Loader) Hash() ([]byte, error)
  • func (l Loader) Load(ctx context.Context) ([]schema.Document, error)
  • func (l Loader) LoadAndSplit(ctx context.Context, splitter textsplitter.TextSplitter) ([]schema.Document, error)
  • func (l Loader) Type() string
• type Option
  • func WithAuto(auto bool) Option
  • func WithSelector(selector string) Option
  • func WithURL(url string) Option
• type RecursiveLoader
  • func NewRecursiveLoader(opts ...RecursiveOption) *RecursiveLoader
  • func (l RecursiveLoader) Hash() ([]byte, error)
  • func (l RecursiveLoader) Load(_ context.Context) ([]schema.Document, error)
  • func (l RecursiveLoader) LoadAndSplit(ctx context.Context, splitter textsplitter.TextSplitter) ([]schema.Document, error)
  • func (l RecursiveLoader) Type() string
• type RecursiveOption
  • func WithMaxPages(maxPages int) RecursiveOption
  • func WithPageLoader(loader *Loader) RecursiveOption

Types

type Content

type Content struct {
	Title    string
	URL      string
	Markdown string
}

Content represents a scraped web page with its title, URL, and markdown content.

type Loader

type Loader struct {
	// contains filtered or unexported fields
}

func NewLoader

func NewLoader(opts ...Option) *Loader

func (Loader) Hash

func (l Loader) Hash() ([]byte, error)

func (Loader) Load

func (l Loader) Load(ctx context.Context) ([]schema.Document, error)

Load loads from a source and returns documents.

func (Loader) LoadAndSplit

func (l Loader) LoadAndSplit(ctx context.Context, splitter textsplitter.TextSplitter) ([]schema.Document, error)

LoadAndSplit loads from a source and splits the documents using a text splitter.

func (Loader) Type

func (l Loader) Type() string

type Option

type Option func(*Loader)

func WithAuto

func WithAuto(auto bool) Option

func WithSelector

func WithSelector(selector string) Option

func WithURL

func WithURL(url string) Option

type RecursiveLoader

type RecursiveLoader struct {
	*Loader
	// contains filtered or unexported fields
}

func NewRecursiveLoader

func NewRecursiveLoader(opts ...RecursiveOption) *RecursiveLoader

func (RecursiveLoader) Hash

func (l RecursiveLoader) Hash() ([]byte, error)

func (RecursiveLoader) Load

func (l RecursiveLoader) Load(_ context.Context) ([]schema.Document, error)

Load loads from a source and returns documents.

func (RecursiveLoader) LoadAndSplit

func (l RecursiveLoader) LoadAndSplit(ctx context.Context, splitter textsplitter.TextSplitter) ([]schema.Document, error)

LoadAndSplit loads from a source and splits the documents using a text splitter.

func (RecursiveLoader) Type

func (l RecursiveLoader) Type() string

type RecursiveOption

type RecursiveOption func(*RecursiveLoader)

func WithMaxPages

func WithMaxPages(maxPages int) RecursiveOption

func WithPageLoader

func WithPageLoader(loader *Loader) RecursiveOption