vision

Documentation

Overview

Package vision provides local image understanding via ONNX-based VLM models (Florence-2). Supports image captioning (describe) and visual question answering (ask) with a shared inference engine.

Index

• Constants
• Variables
• func DefaultModelsDir() string
• func ExpandPrompt(prompt string) string
• func InitModel(modelsDir, variant string, force bool) error
• func IsReady(modelsDir, variant string) bool
• func ListModels()
• func PreprocessImage(path string) ([]float32, error)
• func PreprocessImageFromImage(img image.Image, targetSize int) ([]float32, error)
• func VariantDir(modelsDir, variant string) string
• type Engine
  • func NewEngine(cfg *EngineConfig) (*Engine, error)
  • func (e *Engine) Close()
  • func (e *Engine) Describe(path string) (string, error)
• type EngineConfig
• type ModelFile
  • func ModelFiles(variant string) ([]ModelFile, error)
• type ModelVariant
  • func ResolveModelVariant(id string) (ModelVariant, error)
• type Sampler
  • func DefaultSampler() *Sampler
  • func NewSampler(temperature float64, topK int) *Sampler
  • func (s *Sampler) Sample(logits []float32) int64
• type TaskPrompt
• type Tokenizer
  • func NewTokenizer(vocabPath, mergesPath string) (*Tokenizer, error)
  • func (tk *Tokenizer) Decode(ids []int64) string
  • func (tk *Tokenizer) Encode(text string) []int64
  • func (tk *Tokenizer) EncodeTokens(text string) []string
  • func (tk *Tokenizer) VocabSize() int

Constants

const (
	// InputSize is the expected image dimension (768×768).
	InputSize = 768
	// Channels is the number of color channels.
	Channels = 3
	// HiddenDim is the hidden dimension size for all sub-models.
	HiddenDim = 768
	// NumDecoderLayers is the number of decoder layers.
	NumDecoderLayers = 6
	// NumAttentionHeads is the number of attention heads.
	NumAttentionHeads = 12
	// HeadDim is the dimension per attention head.
	HeadDim = 64
	// ImageSeqLength is the number of visual tokens output by vision_encoder.
	ImageSeqLength = 577
	// MaxTokens is the maximum number of tokens to generate per inference.
	MaxTokens = 512
	// VocabSize is the Florence-2 vocabulary size.
	VocabSize = 51289
	// DecoderStartTokenID is the decoder start token (also EOS).
	DecoderStartTokenID = 2
	// EOSTokenID is the end-of-sequence token ID.
	EOSTokenID = 2
	// PadTokenID is the padding token ID.
	PadTokenID = 1
)

Model constants for Florence-2-base-ft (Heliosoph ONNX export).

const DefaultModelVariant = "base-int8"

Variables

var (
	MeanRGB = [3]float32{0.485, 0.456, 0.406}
	StdRGB  = [3]float32{0.229, 0.224, 0.225}
)

ImageNet normalization parameters (used by Florence-2).

var AvailableModelVariants = []ModelVariant{
	{ID: "base-int8", Description: "Florence-2 Base INT8 (quantized)", Size: "~270 MB"},
}

Functions

func DefaultModelsDir

func DefaultModelsDir() string

func ExpandPrompt

func ExpandPrompt(prompt string) string

ExpandPrompt expands a Florence-2 task token into its full prompt text. Tokens without an expansion (e.g. <VQA>) are returned as-is.

func InitModel

func InitModel(modelsDir, variant string, force bool) error

InitModel downloads all model files for the given variant.

func IsReady

func IsReady(modelsDir, variant string) bool

IsReady checks if all model files exist.

func ListModels

func ListModels()

ListModels prints available model variants to stdout.

func PreprocessImage

func PreprocessImage(path string) ([]float32, error)

PreprocessImage decodes an image file and preprocesses it into a normalized CHW float32 tensor suitable for Florence-2 encoder input.

Steps:

1. Decode image from file
2. Resize to InputSize×InputSize (224×224)
3. Convert to CHW layout
4. Normalize using ImageNet mean/std

func PreprocessImageFromImage

func PreprocessImageFromImage(img image.Image, targetSize int) ([]float32, error)

PreprocessImageFromImage preprocesses a decoded image for Florence-2.

func VariantDir

func VariantDir(modelsDir, variant string) string

Types

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

Engine wraps the four Florence-2 ONNX sub-models into a single inference engine.

func NewEngine

func NewEngine(cfg *EngineConfig) (*Engine, error)

NewEngine creates and initializes the vision engine with all four ONNX sessions.

func (*Engine) Close

func (e *Engine) Close()

func (*Engine) Describe

func (e *Engine) Describe(path string) (string, error)

type EngineConfig

type EngineConfig struct {
	ModelsDir   string
	Variant     string
	LibPath     string
	Tokenizer   *Tokenizer
	MaxTokens   int
	Temperature float64
	TopK        int
}

type ModelFile

type ModelFile struct {
	URL      string
	Filename string
	Size     string
}

ModelFile describes a downloadable model file.

func ModelFiles

func ModelFiles(variant string) ([]ModelFile, error)

ModelFiles returns the list of files to download for a given variant.

type ModelVariant

type ModelVariant struct {
	ID          string
	Description string
	Size        string
}

ModelVariant describes a Florence-2 ONNX model variant.

func ResolveModelVariant

func ResolveModelVariant(id string) (ModelVariant, error)

type Sampler

type Sampler struct {
	Temperature float64
	TopK        int
}

Sampler handles token sampling strategies for autoregressive generation.

func DefaultSampler

func DefaultSampler() *Sampler

DefaultSampler returns a sampler with greedy decoding.

func NewSampler

func NewSampler(temperature float64, topK int) *Sampler

NewSampler creates a sampler with the given parameters. temperature=0 means greedy (always pick highest probability).

func (*Sampler) Sample

func (s *Sampler) Sample(logits []float32) int64

Sample selects the next token ID from logits. When temperature is 0 (or very small), uses greedy selection.

type TaskPrompt

type TaskPrompt string

TaskPrompt is a Florence-2 task token that controls the model's output mode.

const (
	TaskCaption             TaskPrompt = "<CAPTION>"
	TaskDetailedCaption     TaskPrompt = "<DETAILED_CAPTION>"
	TaskMoreDetailedCaption TaskPrompt = "<MORE_DETAILED_CAPTION>"
	TaskOD                  TaskPrompt = "<OD>"
)

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

Tokenizer implements GPT-2's byte-level BPE tokenizer, used by Florence-2.

Usage:

tk, err := NewTokenizer(vocabPath, mergesPath)
ids := tk.Encode("<DETAILED_CAPTION>")
text := tk.Decode(ids)

func NewTokenizer

func NewTokenizer(vocabPath, mergesPath string) (*Tokenizer, error)

NewTokenizer loads a GPT-2 BPE tokenizer from vocab.json and merges.txt. vocabPath is the path to vocab.json, mergesPath is the path to merges.txt.

func (*Tokenizer) Decode

func (tk *Tokenizer) Decode(ids []int64) string

Decode converts a slice of token IDs back to text.

func (*Tokenizer) Encode

func (tk *Tokenizer) Encode(text string) []int64

Encode converts a text string to a slice of token IDs with BOS/EOS.

func (*Tokenizer) EncodeTokens

func (tk *Tokenizer) EncodeTokens(text string) []string

EncodeTokens converts a string to token strings (for debugging).

func (*Tokenizer) VocabSize

func (tk *Tokenizer) VocabSize() int

VocabSize returns the vocabulary size.