parser

package module
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 17, 2026 License: MIT Imports: 43 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	FileTypeUnknown = "unknown"
	FileTypeTXT     = "txt"
	FileTypeMD      = "md"
	FileTypeMDX     = "mdx"
	FileTypeCSV     = "csv"
	FileTypeHTML    = "html"
	FileTypeJSON    = "json"
	FileTypeYAML    = "yaml"
	FileTypeYML     = "yml"
	FileTypeEML     = "eml"
	FileTypeRTF     = "rtf"
	FileTypePDF     = "pdf"
	FileTypePNG     = "png"
	FileTypeJPG     = "jpg"
	FileTypeJPEG    = "jpeg"
	FileTypeDOC     = "doc"
	FileTypeDOCX    = "docx"
	FileTypePPTX    = "pptx"
	FileTypeXLSX    = "xlsx"
	FileTypeEPUB    = "epub"
	FileTypeMHTML   = "mhtml"
	FileTypeMHT     = "mht"
	FileTypeWEBP    = "webp"
	FileTypeGIF     = "gif"
	FileTypeBMP     = "bmp"
	FileTypeTIFF    = "tiff"
	FileTypeTIF     = "tif"
	FileTypeWAV     = "wav"
	FileTypeMP3     = "mp3"
	FileTypeOGG     = "ogg"
	FileTypeFLAC    = "flac"
	FileTypeM4A     = "m4a"
	FileTypeAAC     = "aac"
	FileTypeXML     = "xml"
	FileTypeTOML    = "toml"
	FileTypeTSV     = "tsv"
	FileTypeICS     = "ics"
	FileTypeVCF     = "vcf"
	FileTypeLOG     = "log"
	FileTypeODT     = "odt"
	FileTypeODS     = "ods"
	FileTypeODP     = "odp"
	FileTypePPT     = "ppt"
	FileTypeXLS     = "xls"
	FileTypeSVG     = "svg"
)

Variables

View Source 
var (
	ErrUnsupportedFileType = errors.New("unsupported file type")
	ErrEmptyInput          = errors.New("empty input")
)

Functions

func DetectFileType 

func DetectFileType(req *ParseRequest) string

func SupportedDocumentNotes 

func SupportedDocumentNotes() []string

SupportedDocumentNotes are caveats for operators / UI.

Types

type ASRParser 

type ASRParser struct {

	// SampleRate controls the PCM decode target rate sent to the engine.
	// Defaults to 16000.
	SampleRate int
	// contains filtered or unexported fields
}

ASRParser transcribes audio files using the ling-base recognizer package. It wraps a recognizer.Engine (local, volcengine, qcloud, etc.) and provides a synchronous file-to-text API suitable for the parser router.

The engine must be injected via NewASRParser or SetEngine. If no engine is configured, Parse returns ErrUnsupportedFileType.

func ASRParserFromEnv 

func ASRParserFromEnv() *ASRParser

ASRParserFromEnv creates an ASRParser with a local engine detected from PATH. This is a convenience for simple use cases; for production, inject a specific engine via NewASRParser.

func NewASRParser 

func NewASRParser(engine base.Engine) *ASRParser

NewASRParser creates an ASRParser backed by the given recognizer Engine.

func (*ASRParser) Parse 

func (p *ASRParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*ASRParser) Provider 

func (p *ASRParser) Provider() string

func (*ASRParser) SetEngine 

func (p *ASRParser) SetEngine(engine base.Engine)

SetEngine replaces the underlying recognizer engine.

func (*ASRParser) SupportedTypes 

func (p *ASRParser) SupportedTypes() []string

type CSVParser 

type CSVParser struct{}

func (*CSVParser) Parse 

func (p *CSVParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*CSVParser) Provider 

func (p *CSVParser) Provider() string

func (*CSVParser) SupportedTypes 

func (p *CSVParser) SupportedTypes() []string

type DOCParser 

type DOCParser struct{}

func (*DOCParser) Parse 

func (p *DOCParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*DOCParser) Provider 

func (p *DOCParser) Provider() string

func (*DOCParser) SupportedTypes 

func (p *DOCParser) SupportedTypes() []string

type DOCXParser 

type DOCXParser struct{}

func (*DOCXParser) Parse 

func (p *DOCXParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*DOCXParser) Provider 

func (p *DOCXParser) Provider() string

func (*DOCXParser) SupportedTypes 

func (p *DOCXParser) SupportedTypes() []string

type EMLParser 

type EMLParser struct{}

func (*EMLParser) Parse 

func (p *EMLParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*EMLParser) Provider 

func (p *EMLParser) Provider() string

func (*EMLParser) SupportedTypes 

func (p *EMLParser) SupportedTypes() []string

type EPUBParser 

type EPUBParser struct{}

func (*EPUBParser) Parse 

func (p *EPUBParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*EPUBParser) Provider 

func (p *EPUBParser) Provider() string

func (*EPUBParser) SupportedTypes 

func (p *EPUBParser) SupportedTypes() []string

type HTMLParser 

type HTMLParser struct{}

func (*HTMLParser) Parse 

func (p *HTMLParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*HTMLParser) Provider 

func (p *HTMLParser) Provider() string

func (*HTMLParser) SupportedTypes 

func (p *HTMLParser) SupportedTypes() []string

type ICSParser 

type ICSParser struct{}

ICSParser parses iCalendar (.ics) files and extracts event summaries, descriptions, dates, and locations.

func (*ICSParser) Parse 

func (p *ICSParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*ICSParser) Provider 

func (p *ICSParser) Provider() string

func (*ICSParser) SupportedTypes 

func (p *ICSParser) SupportedTypes() []string

type JSONParser 

type JSONParser struct{}

func (*JSONParser) Parse 

func (p *JSONParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*JSONParser) Provider 

func (p *JSONParser) Provider() string

func (*JSONParser) SupportedTypes 

func (p *JSONParser) SupportedTypes() []string

type LOGParser 

type LOGParser struct{}

LOGParser parses log files. Log files are text-based but may have structured entries (timestamps, levels). The parser preserves line structure and optionally extracts key fields.

func (*LOGParser) Parse 

func (p *LOGParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*LOGParser) Provider 

func (p *LOGParser) Provider() string

func (*LOGParser) SupportedTypes 

func (p *LOGParser) SupportedTypes() []string

type MDXParser 

type MDXParser struct{}

MDXParser parses MDX (Markdown with JSX) files. MDX is a format that allows JSX components within Markdown.

func (*MDXParser) Parse 

func (p *MDXParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*MDXParser) Provider 

func (p *MDXParser) Provider() string

func (*MDXParser) SupportedTypes 

func (p *MDXParser) SupportedTypes() []string

type MHTMLParser 

type MHTMLParser struct{}

func (*MHTMLParser) Parse 

func (p *MHTMLParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*MHTMLParser) Provider 

func (p *MHTMLParser) Provider() string

func (*MHTMLParser) SupportedTypes 

func (p *MHTMLParser) SupportedTypes() []string

type MarkdownParser 

type MarkdownParser struct {
	// contains filtered or unexported fields
}

MarkdownParser parses standard Markdown files (alias for TXT parser with MD support).

func NewMarkdownParser 

func NewMarkdownParser() *MarkdownParser

func (*MarkdownParser) Parse 

func (p *MarkdownParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*MarkdownParser) Provider 

func (p *MarkdownParser) Provider() string

func (*MarkdownParser) SupportedTypes 

func (p *MarkdownParser) SupportedTypes() []string

type OCRParser 

type OCRParser struct {
	// Language is a language hint passed to the OCR provider.
	Language string
	// Driver overrides the global provider for this parser instance.
	// If nil, the global provider (set via ocr.SetProvider) is used.
	Driver ocr.Provider
}

OCRParser implements the Parser interface for image files using a cloud OCR provider. If no OCR provider is registered (via the ocr package), Parse returns ErrUnsupportedFileType.

func (*OCRParser) Parse 

func (p *OCRParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*OCRParser) Provider 

func (p *OCRParser) Provider() string

func (*OCRParser) SupportedTypes 

func (p *OCRParser) SupportedTypes() []string

type ODTParser 

type ODTParser struct{}

ODTParser parses OpenDocument Text (.odt) files, which are zip archives containing XML content. It extracts text from content.xml.

func (*ODTParser) Parse 

func (p *ODTParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*ODTParser) Provider 

func (p *ODTParser) Provider() string

func (*ODTParser) SupportedTypes 

func (p *ODTParser) SupportedTypes() []string

type PDFParser 

type PDFParser struct{}

func (*PDFParser) Parse 

func (p *PDFParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*PDFParser) Provider 

func (p *PDFParser) Provider() string

func (*PDFParser) SupportedTypes 

func (p *PDFParser) SupportedTypes() []string

type PPTXParser 

type PPTXParser struct{}

func (*PPTXParser) Parse 

func (p *PPTXParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*PPTXParser) Provider 

func (p *PPTXParser) Provider() string

func (*PPTXParser) SupportedTypes 

func (p *PPTXParser) SupportedTypes() []string

type ParseOptions 

type ParseOptions struct {
	MaxTextLength      int
	IncludeTables      bool
	IncludeHidden      bool
	PreserveLineBreaks bool
}

type ParsePreviewMeta 

type ParsePreviewMeta struct {
	Format    string         `json:"format,omitempty"`
	CharCount int            `json:"charCount,omitempty"`
	Preview   string         `json:"preview,omitempty"`
	Metadata  map[string]any `json:"metadata,omitempty"`
}

ParsePreviewMeta holds parse-stage metadata for the ingest preview UI.

func ParseBytesWithMeta 

func ParseBytesWithMeta(ctx context.Context, fileName string, content []byte, opts *ParseOptions) (string, *ParsePreviewMeta, error)

type ParseRequest 

type ParseRequest struct {
	FileType    string
	FileName    string
	Path        string
	ContentType string
	Content     []byte
	Reader      io.Reader
	Metadata    map[string]any
}

type ParseResult 

type ParseResult struct {
	FileType string
	FileName string
	Text     string
	Sections []Section
	Metadata map[string]any
	ParsedAt time.Time
}

func ParseAuto 

func ParseAuto(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func ParseBytes 

func ParseBytes(ctx context.Context, fileName string, content []byte, opts *ParseOptions) (*ParseResult, error)

func ParsePath 

func ParsePath(ctx context.Context, path string, opts *ParseOptions) (*ParseResult, error)

type Parser 

type Parser interface {
	Provider() string
	SupportedTypes() []string
	Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)
}

type RTFParser 

type RTFParser struct{}

func (*RTFParser) Parse 

func (p *RTFParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*RTFParser) Provider 

func (p *RTFParser) Provider() string

func (*RTFParser) SupportedTypes 

func (p *RTFParser) SupportedTypes() []string

type Router 

type Router struct {
	// contains filtered or unexported fields
}

func DefaultRouter 

func DefaultRouter() *Router

func NewRouter 

func NewRouter(parsers ...Parser) *Router

func (*Router) Parse 

func (r *Router) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*Router) Register 

func (r *Router) Register(p Parser) error

type SVGParser 

type SVGParser struct{}

SVGParser extracts text content from SVG (Scalable Vector Graphics) files. SVG is XML-based; this parser extracts text from <text>, <tspan>, and <title> elements.

func (*SVGParser) Parse 

func (p *SVGParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*SVGParser) Provider 

func (p *SVGParser) Provider() string

func (*SVGParser) SupportedTypes 

func (p *SVGParser) SupportedTypes() []string

type Section 

type Section struct {
	Type     SectionType
	Index    int
	Title    string
	Text     string
	Metadata map[string]any
}

type SectionType 

type SectionType string
const (
	SectionTypeUnknown  SectionType = "unknown"
	SectionTypeDocument SectionType = "document"
	SectionTypePage     SectionType = "page"
	SectionTypeSheet    SectionType = "sheet"
	SectionTypeSlide    SectionType = "slide"
)

type SupportedDocumentFormat 

type SupportedDocumentFormat struct {
	Extension   string `json:"extension"`
	Description string `json:"description"`
}

SupportedDocumentFormat is one file type the upload pipeline can parse (see DetectFileType / DefaultRouter).

func SupportedDocumentFormats 

func SupportedDocumentFormats() []SupportedDocumentFormat

SupportedDocumentFormats lists extensions accepted for knowledge upload.

type TOMLParser 

type TOMLParser struct{}

TOMLParser parses TOML configuration files and renders them as key=value text.

func (*TOMLParser) Parse 

func (p *TOMLParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*TOMLParser) Provider 

func (p *TOMLParser) Provider() string

func (*TOMLParser) SupportedTypes 

func (p *TOMLParser) SupportedTypes() []string

type TSVParser 

type TSVParser struct{}

TSVParser parses tab-separated value files.

func (*TSVParser) Parse 

func (p *TSVParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*TSVParser) Provider 

func (p *TSVParser) Provider() string

func (*TSVParser) SupportedTypes 

func (p *TSVParser) SupportedTypes() []string

type TXTParser 

type TXTParser struct{}

func (*TXTParser) Parse 

func (p *TXTParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*TXTParser) Provider 

func (p *TXTParser) Provider() string

func (*TXTParser) SupportedTypes 

func (p *TXTParser) SupportedTypes() []string

type VCFParser 

type VCFParser struct{}

VCFParser parses vCard (.vcf) contact files and extracts contact information.

func (*VCFParser) Parse 

func (p *VCFParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*VCFParser) Provider 

func (p *VCFParser) Provider() string

func (*VCFParser) SupportedTypes 

func (p *VCFParser) SupportedTypes() []string

type XLSXParser 

type XLSXParser struct{}

func (*XLSXParser) Parse 

func (p *XLSXParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*XLSXParser) Provider 

func (p *XLSXParser) Provider() string

func (*XLSXParser) SupportedTypes 

func (p *XLSXParser) SupportedTypes() []string

type XMLParser 

type XMLParser struct{}

XMLParser extracts text content from XML files by walking the element tree and collecting character data.

func (*XMLParser) Parse 

func (p *XMLParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*XMLParser) Provider 

func (p *XMLParser) Provider() string

func (*XMLParser) SupportedTypes 

func (p *XMLParser) SupportedTypes() []string

type YAMLParser 

type YAMLParser struct{}

func (*YAMLParser) Parse 

func (p *YAMLParser) Parse(ctx context.Context, req *ParseRequest, opts *ParseOptions) (*ParseResult, error)

func (*YAMLParser) Provider 

func (p *YAMLParser) Provider() string

func (*YAMLParser) SupportedTypes 

func (p *YAMLParser) SupportedTypes() []string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.