understanding

package
v0.0.0-...-5c6d8ba Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 10, 2026 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	WarningQualityDegraded = "" /* 203-byte string literal not displayed */
	DefaultMaxTextChars    = 12000
)
View Source 
const (
	MIMEDOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
	MIMEXLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
	MIMEPPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
)

Variables

This section is empty.

Functions

func CacheableArtifactJSON 

func CacheableArtifactJSON(artifact *Artifact) string

func CanonicalMIME 

func CanonicalMIME(mimeType string, data []byte) string

func HumanFilename 

func HumanFilename(filename string) string

func MetadataJSON 

func MetadataJSON(metadata map[string]any) string

func RebindArtifactProvenance 

func RebindArtifactProvenance(artifact *Artifact, input Input)

Types

type Artifact 

type Artifact struct {
	Kind       ArtifactKind   `json:"kind,omitempty"`
	Processor  string         `json:"processor,omitempty"`
	Version    string         `json:"version,omitempty"`
	MimeType   string         `json:"mime_type,omitempty"`
	Text       string         `json:"text,omitempty"`
	Warning    string         `json:"warning,omitempty"`
	Metadata   map[string]any `json:"metadata,omitempty"`
	Provenance []Provenance   `json:"provenance,omitempty"`
	Chunks     []Chunk        `json:"chunks,omitempty"`
	Embeddings []Embedding    `json:"embeddings,omitempty"`
	Pages      []Page         `json:"pages,omitempty"`
	Tables     []Table        `json:"tables,omitempty"`
	Forms      []FormField    `json:"forms,omitempty"`
}

func ArtifactFromJSON 

func ArtifactFromJSON(raw string) (*Artifact, bool)

func ProcessWithProcessor 

func ProcessWithProcessor(ctx context.Context, processor Processor, input Input) (*Artifact, error)

type ArtifactKind 

type ArtifactKind string
const (
	ArtifactText          ArtifactKind = "text"
	ArtifactMetadata      ArtifactKind = "metadata"
	ArtifactOCRText       ArtifactKind = "ocr_text"
	ArtifactLayout        ArtifactKind = "layout"
	ArtifactTable         ArtifactKind = "table"
	ArtifactForm          ArtifactKind = "form"
	ArtifactChunk         ArtifactKind = "chunk"
	ArtifactImageCaption  ArtifactKind = "image_caption"
	ArtifactImageMetadata ArtifactKind = "image_metadata"
	ArtifactEmbedding     ArtifactKind = "embedding"
	ArtifactRaw           ArtifactKind = "raw"
)

type Block 

type Block struct {
	ID         string         `json:"id,omitempty"`
	Kind       string         `json:"kind,omitempty"`
	Text       string         `json:"text,omitempty"`
	Confidence float64        `json:"confidence,omitempty"`
	Bounds     *Bounds        `json:"bounds,omitempty"`
	Provenance Provenance     `json:"provenance,omitempty"`
	Metadata   map[string]any `json:"metadata,omitempty"`
}

type Bounds 

type Bounds struct {
	Left   float64 `json:"left,omitempty"`
	Top    float64 `json:"top,omitempty"`
	Width  float64 `json:"width,omitempty"`
	Height float64 `json:"height,omitempty"`
	Unit   string  `json:"unit,omitempty"`
}

type Chunk 

type Chunk struct {
	ID            string         `json:"id,omitempty"`
	Text          string         `json:"text,omitempty"`
	StartChar     int            `json:"start_char,omitempty"`
	EndChar       int            `json:"end_char,omitempty"`
	TokenEstimate int            `json:"token_estimate,omitempty"`
	Source        Provenance     `json:"source,omitempty"`
	Metadata      map[string]any `json:"metadata,omitempty"`
}

type ChunkOptions 

type ChunkOptions struct {
	Enabled      bool
	MaxChars     int
	OverlapChars int
}

type ChunkProcessor 

type ChunkProcessor struct {
	MaxChars     int
	OverlapChars int
}

func (ChunkProcessor) Descriptor 

func (p ChunkProcessor) Descriptor() ProcessorDescriptor

func (ChunkProcessor) Name 

func (p ChunkProcessor) Name() string

func (ChunkProcessor) Process 

func (p ChunkProcessor) Process(_ context.Context, input Input) (*Artifact, error)

func (ChunkProcessor) Supports 

func (p ChunkProcessor) Supports(mimeType string) bool

func (ChunkProcessor) Version 

func (p ChunkProcessor) Version() string

type CostTier 

type CostTier string
const (
	CostFree   CostTier = "free"
	CostLow    CostTier = "low"
	CostMedium CostTier = "medium"
	CostHigh   CostTier = "high"
)

type DescribedProcessor 

type DescribedProcessor interface {
	Descriptor() ProcessorDescriptor
}

type Embedding 

type Embedding struct {
	ChunkID    string     `json:"chunk_id,omitempty"`
	Model      string     `json:"model,omitempty"`
	Dimensions int        `json:"dimensions,omitempty"`
	Vector     []float64  `json:"vector,omitempty"`
	Source     Provenance `json:"source,omitempty"`
}

type ErrUnsupportedMIME 

type ErrUnsupportedMIME struct {
	MIMEType string
}

func (ErrUnsupportedMIME) Error 

func (e ErrUnsupportedMIME) Error() string

type FileClass 

type FileClass string
const (
	FileClassText               FileClass = "text"
	FileClassImage              FileClass = "image"
	FileClassPDF                FileClass = "pdf"
	FileClassOfficeDocument     FileClass = "office_document"
	FileClassOfficeSpreadsheet  FileClass = "office_spreadsheet"
	FileClassOfficePresentation FileClass = "office_presentation"
	FileClassAudio              FileClass = "audio"
	FileClassVideo              FileClass = "video"
	FileClassArchive            FileClass = "archive"
	FileClassUnknown            FileClass = "unknown"
)

func ClassifyMIME 

func ClassifyMIME(mimeType string) FileClass

type FormField 

type FormField struct {
	Key        string         `json:"key,omitempty"`
	Value      string         `json:"value,omitempty"`
	Page       int            `json:"page,omitempty"`
	Confidence float64        `json:"confidence,omitempty"`
	Bounds     *Bounds        `json:"bounds,omitempty"`
	Provenance Provenance     `json:"provenance,omitempty"`
	Metadata   map[string]any `json:"metadata,omitempty"`
}

type ImageMetadataProcessor 

type ImageMetadataProcessor struct{}

func (ImageMetadataProcessor) Descriptor 

func (ImageMetadataProcessor) Descriptor() ProcessorDescriptor

func (ImageMetadataProcessor) Name 

func (ImageMetadataProcessor) Name() string

func (ImageMetadataProcessor) Process 

func (ImageMetadataProcessor) Process(_ context.Context, input Input) (*Artifact, error)

func (ImageMetadataProcessor) Supports 

func (ImageMetadataProcessor) Supports(mimeType string) bool

func (ImageMetadataProcessor) Version 

func (ImageMetadataProcessor) Version() string

type Input 

type Input struct {
	FileID       string
	Sha256       string
	Filename     string
	MimeType     string
	Size         int64
	Data         []byte
	MaxTextChars int
	DerivedText  string
	Artifacts    []Artifact
}

type LatencyTier 

type LatencyTier string
const (
	LatencyLocalFast LatencyTier = "local_fast"
	LatencyLocalSlow LatencyTier = "local_slow"
	LatencyRemote    LatencyTier = "remote"
	LatencyAsync     LatencyTier = "async"
)

type OOXMLTextProcessor 

type OOXMLTextProcessor struct{}

func (OOXMLTextProcessor) Descriptor 

func (OOXMLTextProcessor) Descriptor() ProcessorDescriptor

func (OOXMLTextProcessor) Name 

func (OOXMLTextProcessor) Name() string

func (OOXMLTextProcessor) Process 

func (OOXMLTextProcessor) Process(_ context.Context, input Input) (*Artifact, error)

func (OOXMLTextProcessor) Supports 

func (OOXMLTextProcessor) Supports(mimeType string) bool

func (OOXMLTextProcessor) Version 

func (OOXMLTextProcessor) Version() string

type PDFTextProcessor 

type PDFTextProcessor struct{}

func (PDFTextProcessor) Descriptor 

func (PDFTextProcessor) Descriptor() ProcessorDescriptor

func (PDFTextProcessor) Name 

func (PDFTextProcessor) Name() string

func (PDFTextProcessor) Process 

func (PDFTextProcessor) Process(_ context.Context, input Input) (*Artifact, error)

func (PDFTextProcessor) Supports 

func (PDFTextProcessor) Supports(mimeType string) bool

func (PDFTextProcessor) Version 

func (PDFTextProcessor) Version() string

type Page 

type Page struct {
	Number     int            `json:"number,omitempty"`
	Text       string         `json:"text,omitempty"`
	Width      float64        `json:"width,omitempty"`
	Height     float64        `json:"height,omitempty"`
	Unit       string         `json:"unit,omitempty"`
	Blocks     []Block        `json:"blocks,omitempty"`
	Provenance Provenance     `json:"provenance,omitempty"`
	Metadata   map[string]any `json:"metadata,omitempty"`
}

type PipelineOptions 

type PipelineOptions struct {
	PlanOptions
	Chunking ChunkOptions
}

type PipelineResult 

type PipelineResult struct {
	Plan      *Plan
	Artifacts []Artifact
}

func ProcessPipeline 

func ProcessPipeline(ctx context.Context, registry *Registry, input Input, opts PipelineOptions) (*PipelineResult, error)

type Plan 

type Plan struct {
	MimeType string
	Class    FileClass
	Profile  ProcessingProfile
	Steps    []PlanStep
	Warnings []string
}

func (*Plan) PrimaryProcessor 

func (p *Plan) PrimaryProcessor() Processor

type PlanOptions 

type PlanOptions struct {
	Profile              ProcessingProfile
	RequiredCapabilities []ProcessorCapability
	PreferredArtifacts   []ArtifactKind
	AllowNetwork         bool
}

func DefaultPlanOptionsForDerivedContext 

func DefaultPlanOptionsForDerivedContext(mimeType string, profile ProcessingProfile) PlanOptions

type PlanStep 

type PlanStep struct {
	Processor  Processor
	Descriptor ProcessorDescriptor
	Reason     string
}

type ProcessingProfile 

type ProcessingProfile string
const (
	ProfileFast     ProcessingProfile = "fast"
	ProfileBalanced ProcessingProfile = "balanced"
	ProfileQuality  ProcessingProfile = "quality"
)

func NormalizeProfile 

func NormalizeProfile(profile string) ProcessingProfile

type Processor 

type Processor interface {
	Name() string
	Version() string
	Supports(mimeType string) bool
	Process(ctx context.Context, input Input) (*Artifact, error)
}

type ProcessorCapability 

type ProcessorCapability string
const (
	CapabilityDetect       ProcessorCapability = "detect"
	CapabilityTextExtract  ProcessorCapability = "text_extract"
	CapabilityMetadata     ProcessorCapability = "metadata"
	CapabilityOCR          ProcessorCapability = "ocr"
	CapabilityLayout       ProcessorCapability = "layout"
	CapabilityTableExtract ProcessorCapability = "table_extract"
	CapabilityFormExtract  ProcessorCapability = "form_extract"
	CapabilityImageCaption ProcessorCapability = "image_caption"
	CapabilityChunk        ProcessorCapability = "chunk"
	CapabilityEmbed        ProcessorCapability = "embed"
)

type ProcessorDescriptor 

type ProcessorDescriptor struct {
	Name            string
	Version         string
	Backend         string
	Description     string
	FileClasses     []FileClass
	MIMETypes       []string
	ArtifactKinds   []ArtifactKind
	Capabilities    []ProcessorCapability
	Profiles        []ProcessingProfile
	LatencyTier     LatencyTier
	CostTier        CostTier
	Deterministic   bool
	Lossy           bool
	RequiresNetwork bool
	Priority        int
	Notes           []string
}

func DescribeProcessor 

func DescribeProcessor(processor Processor) ProcessorDescriptor

type Provenance 

type Provenance struct {
	FileID       string `json:"file_id,omitempty"`
	Sha256       string `json:"sha256,omitempty"`
	Filename     string `json:"filename,omitempty"`
	MimeType     string `json:"mime_type,omitempty"`
	Processor    string `json:"processor,omitempty"`
	ArtifactKind string `json:"artifact_kind,omitempty"`
	Page         int    `json:"page,omitempty"`
	BlockID      string `json:"block_id,omitempty"`
	StartChar    int    `json:"start_char,omitempty"`
	EndChar      int    `json:"end_char,omitempty"`
}

type Registry 

type Registry struct {
	// contains filtered or unexported fields
}

func NewDefaultRegistry 

func NewDefaultRegistry() *Registry

func NewRegistry 

func NewRegistry(processors ...Processor) *Registry

func NewRegistryWithRemoteProcessors 

func NewRegistryWithRemoteProcessors(processors ...Processor) *Registry

func (*Registry) Plan 

func (r *Registry) Plan(input Input, opts PlanOptions) (*Plan, error)

func (*Registry) Process 

func (r *Registry) Process(ctx context.Context, input Input) (*Artifact, error)

func (*Registry) ProcessorFor 

func (r *Registry) ProcessorFor(mimeType string) Processor

type RemoteProcessor 

type RemoteProcessor struct {
	// contains filtered or unexported fields
}

func NewRemoteProcessor 

func NewRemoteProcessor(cfg RemoteProcessorConfig) (*RemoteProcessor, error)

func (*RemoteProcessor) Descriptor 

func (p *RemoteProcessor) Descriptor() ProcessorDescriptor

func (*RemoteProcessor) Name 

func (p *RemoteProcessor) Name() string

func (*RemoteProcessor) Process 

func (p *RemoteProcessor) Process(ctx context.Context, input Input) (*Artifact, error)

func (*RemoteProcessor) Supports 

func (p *RemoteProcessor) Supports(mimeType string) bool

func (*RemoteProcessor) Version 

func (p *RemoteProcessor) Version() string

type RemoteProcessorConfig 

type RemoteProcessorConfig struct {
	Name            string
	Version         string
	Backend         string
	Endpoint        string
	Method          string
	Headers         map[string]string
	MIMETypes       []string
	FileClasses     []FileClass
	ArtifactKinds   []ArtifactKind
	Capabilities    []ProcessorCapability
	Profiles        []ProcessingProfile
	Timeout         time.Duration
	Priority        int
	RequiresNetwork bool
	Description     string
	Notes           []string
}

type Table 

type Table struct {
	ID         string         `json:"id,omitempty"`
	Page       int            `json:"page,omitempty"`
	Rows       [][]string     `json:"rows,omitempty"`
	Bounds     *Bounds        `json:"bounds,omitempty"`
	Confidence float64        `json:"confidence,omitempty"`
	Provenance Provenance     `json:"provenance,omitempty"`
	Metadata   map[string]any `json:"metadata,omitempty"`
}

type TextProcessor 

type TextProcessor struct{}

func (TextProcessor) Descriptor 

func (TextProcessor) Descriptor() ProcessorDescriptor

func (TextProcessor) Name 

func (TextProcessor) Name() string

func (TextProcessor) Process 

func (TextProcessor) Process(_ context.Context, input Input) (*Artifact, error)

func (TextProcessor) Supports 

func (TextProcessor) Supports(mimeType string) bool

func (TextProcessor) Version 

func (TextProcessor) Version() string

type TikaProcessor 

type TikaProcessor struct {
	// contains filtered or unexported fields
}

func NewTikaProcessor 

func NewTikaProcessor(cfg TikaProcessorConfig) (*TikaProcessor, error)

func (*TikaProcessor) Descriptor 

func (p *TikaProcessor) Descriptor() ProcessorDescriptor

func (*TikaProcessor) Name 

func (p *TikaProcessor) Name() string

func (*TikaProcessor) Process 

func (p *TikaProcessor) Process(ctx context.Context, input Input) (*Artifact, error)

func (*TikaProcessor) Supports 

func (p *TikaProcessor) Supports(mimeType string) bool

func (*TikaProcessor) Version 

func (p *TikaProcessor) Version() string

type TikaProcessorConfig 

type TikaProcessorConfig struct {
	Name      string
	Version   string
	Endpoint  string
	Headers   map[string]string
	MIMETypes []string
	Profiles  []ProcessingProfile
	Timeout   time.Duration
	Priority  int
	OCR       bool
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.