documentloaders

Documentation

Overview

Package documentloaders implements document loaders that turn various file formats into text documents, mirroring langchain's document_loaders module.

Index

• type Directory
  • func (d *Directory) Load(ctx context.Context) ([]Document, error)
• type Document
• type JSONLoader
  • func (j *JSONLoader) Load(_ context.Context) ([]Document, error)
• type Loader
• type LoaderFunc
  • func (f LoaderFunc) Load(ctx context.Context) ([]Document, error)
• type Markdown
  • func (m *Markdown) Load(_ context.Context) ([]Document, error)
• type Text
  • func (t *Text) Load(_ context.Context) ([]Document, error)
• type TextReader
  • func (r *TextReader) Load(_ context.Context) ([]Document, error)

Types

type Directory

type Directory struct {
	Path       string
	Extensions []string
	LoaderFor  func(path string) Loader
}

Directory loads all files of matching extensions under a directory recursively as individual documents.

func (*Directory) Load

func (d *Directory) Load(ctx context.Context) ([]Document, error)

Load implements Loader.

type Document

type Document struct {
	// PageContent is the extracted text.
	PageContent string
	// Metadata holds source info such as the file path.
	Metadata map[string]any
}

Document is a single loaded document.

type JSONLoader

type JSONLoader struct {
	Path            string
	TextContentPath string
}

JSONLoader loads a JSON file. If the root is an array, each element becomes a document; TextContentPath selects the string field to use as page content.

func (*JSONLoader) Load

func (j *JSONLoader) Load(_ context.Context) ([]Document, error)

Load implements Loader.

type Loader

type Loader interface {
	// Load returns all documents.
	Load(ctx context.Context) ([]Document, error)
}

Loader loads documents from a source.

type LoaderFunc

type LoaderFunc func(ctx context.Context) ([]Document, error)

LoaderFunc adapts a function into a Loader.

func (LoaderFunc) Load

func (f LoaderFunc) Load(ctx context.Context) ([]Document, error)

Load implements Loader.

type Markdown

type Markdown struct {
	Path string
}

Markdown loads a markdown file, preserving its text.

func (*Markdown) Load

func (m *Markdown) Load(_ context.Context) ([]Document, error)

Load implements Loader.

type Text

type Text struct {
	Path string
}

Text loads a plain-text file.

func (*Text) Load

func (t *Text) Load(_ context.Context) ([]Document, error)

Load implements Loader.

type TextReader

type TextReader struct {
	Reader io.Reader
	Source string
}

TextReader loads text from a reader and an optional source name.

func (*TextReader) Load

func (r *TextReader) Load(_ context.Context) ([]Document, error)

Load implements Loader.