inference

Documentation

Index

• type Downloader
  • func NewDownloader(modelsDir string) *Downloader
  • func (d *Downloader) DownloadModel(ctx context.Context, modelID string, url string) error
  • func (d *Downloader) GetProgress(modelID string) (float64, bool)
• type GenOptions
• type InferenceProvider
• type InferenceRequest
• type InferenceResponse
• type InferenceService
  • func NewInferenceService(provider InferenceProvider) *InferenceService
  • func (s *InferenceService) Execute(ctx context.Context, req InferenceRequest) (*InferenceResponse, error)
• type LMStudioProvider
  • func NewLMStudioProvider(baseURL string) *LMStudioProvider
  • func (p *LMStudioProvider) Generate(ctx context.Context, prompt string, options GenOptions) (string, error)
  • func (p *LMStudioProvider) ListModels(ctx context.Context) ([]string, error)
  • func (p *LMStudioProvider) PullModel(ctx context.Context, modelID string) error
  • func (p *LMStudioProvider) TestConnection(ctx context.Context) (string, error)
• type ModelInfo
• type OllamaProvider
  • func NewOllamaProvider(baseURL string) *OllamaProvider
  • func (p *OllamaProvider) Generate(ctx context.Context, prompt string, options GenOptions) (string, error)
  • func (p *OllamaProvider) ListModels(ctx context.Context) ([]string, error)
  • func (p *OllamaProvider) PullModel(ctx context.Context, modelID string) error
  • func (p *OllamaProvider) TestConnection(ctx context.Context) (string, error)
• type OpenAIProvider
  • func NewOpenAIProvider(baseURL string) *OpenAIProvider
  • func (p *OpenAIProvider) Generate(ctx context.Context, prompt string, options GenOptions) (string, error)
  • func (p *OpenAIProvider) ListModels(ctx context.Context) ([]string, error)
  • func (p *OpenAIProvider) PullModel(ctx context.Context, modelID string) error
  • func (p *OpenAIProvider) TestConnection(ctx context.Context) (string, error)
• type WriteCounter
  • func (wc *WriteCounter) Write(p []byte) (int, error)

Types

type Downloader

type Downloader struct {
	// contains filtered or unexported fields
}

Downloader handles background downloading of models

func NewDownloader

func NewDownloader(modelsDir string) *Downloader

NewDownloader creates a new model downloader

func (*Downloader) DownloadModel

func (d *Downloader) DownloadModel(ctx context.Context, modelID string, url string) error

DownloadModel downloads a model from a URL

func (*Downloader) GetProgress

func (d *Downloader) GetProgress(modelID string) (float64, bool)

GetProgress returns the download progress for a model (0-1)

type GenOptions

type GenOptions struct {
	ModelID     string  `json:"modelId"`
	MaxTokens   int     `json:"max_tokens"`
	Temperature float32 `json:"temperature"`
}

GenOptions represents the parameters for a generation request

type InferenceProvider

type InferenceProvider interface {
	// Generate performs a full generation request to the provider
	Generate(ctx context.Context, prompt string, options GenOptions) (string, error)
	// TestConnection verifies that the provider is reachable and functional
	TestConnection(ctx context.Context) (string, error)
	// ListModels retrieves a list of available models from the provider
	ListModels(ctx context.Context) ([]string, error)
	// PullModel requests the provider to download a new model
	PullModel(ctx context.Context, modelID string) error
}

InferenceProvider defines the interface for external inference engines

type InferenceRequest

type InferenceRequest struct {
	ModelID     string  `json:"modelId"`
	Prompt      string  `json:"prompt"`
	Context     string  `json:"context"`
	MaxTokens   int     `json:"max_tokens"`
	Temperature float32 `json:"temperature"`
}

InferenceRequest represents a request to the Local SLM

type InferenceResponse

type InferenceResponse struct {
	Text  string `json:"text"`
	Error string `json:"error,omitempty"`
}

InferenceResponse represents the distilled output from Local SLM

type InferenceService

type InferenceService struct {
	// contains filtered or unexported fields
}

InferenceService manages the external inference execution

func NewInferenceService

func NewInferenceService(provider InferenceProvider) *InferenceService

NewInferenceService creates a new inference service

func (*InferenceService) Execute

func (s *InferenceService) Execute(ctx context.Context, req InferenceRequest) (*InferenceResponse, error)

Execute performs inference using the configured provider

type LMStudioProvider

type LMStudioProvider struct {
	BaseURL string
	Client  *http.Client
}

LMStudioProvider implements InferenceProvider for LM Studio Native API (/api/v1)

func NewLMStudioProvider

func NewLMStudioProvider(baseURL string) *LMStudioProvider

NewLMStudioProvider creates a new LM Studio native provider

func (*LMStudioProvider) Generate

func (p *LMStudioProvider) Generate(ctx context.Context, prompt string, options GenOptions) (string, error)

func (*LMStudioProvider) ListModels

func (p *LMStudioProvider) ListModels(ctx context.Context) ([]string, error)

func (*LMStudioProvider) PullModel

func (p *LMStudioProvider) PullModel(ctx context.Context, modelID string) error

func (*LMStudioProvider) TestConnection

func (p *LMStudioProvider) TestConnection(ctx context.Context) (string, error)

type ModelInfo

type ModelInfo struct {
	ID          string `json:"id"`
	Name        string `json:"name"`
	Size        int64  `json:"size"`
	Path        string `json:"path"`
	Downloaded  bool   `json:"downloaded"`
	DownloadURL string `json:"downloadUrl"`
	Description string `json:"description"`
	LastUsed    string `json:"lastUsed"`
}

ModelInfo represents metadata about a local model

type OllamaProvider

type OllamaProvider struct {
	BaseURL string
	Client  *http.Client
}

OllamaProvider implements InferenceProvider for Ollama native API

func NewOllamaProvider

func NewOllamaProvider(baseURL string) *OllamaProvider

NewOllamaProvider creates a new Ollama provider

func (*OllamaProvider) Generate

func (p *OllamaProvider) Generate(ctx context.Context, prompt string, options GenOptions) (string, error)

func (*OllamaProvider) ListModels

func (p *OllamaProvider) ListModels(ctx context.Context) ([]string, error)

func (*OllamaProvider) PullModel

func (p *OllamaProvider) PullModel(ctx context.Context, modelID string) error

func (*OllamaProvider) TestConnection

func (p *OllamaProvider) TestConnection(ctx context.Context) (string, error)

type OpenAIProvider

type OpenAIProvider struct {
	BaseURL string
	Client  *http.Client
}

OpenAIProvider implements InferenceProvider for OpenAI-compatible APIs

func NewOpenAIProvider

func NewOpenAIProvider(baseURL string) *OpenAIProvider

NewOpenAIProvider creates a new OpenAI compatible provider

func (*OpenAIProvider) Generate

func (p *OpenAIProvider) Generate(ctx context.Context, prompt string, options GenOptions) (string, error)

func (*OpenAIProvider) ListModels

func (p *OpenAIProvider) ListModels(ctx context.Context) ([]string, error)

func (*OpenAIProvider) PullModel

func (p *OpenAIProvider) PullModel(ctx context.Context, modelID string) error

func (*OpenAIProvider) TestConnection

func (p *OpenAIProvider) TestConnection(ctx context.Context) (string, error)

type WriteCounter

type WriteCounter struct {
	Total      int64
	Downloaded int64
	OnProgress func(float64)
}

WriteCounter counts the number of bytes written and updates progress

func (*WriteCounter) Write

func (wc *WriteCounter) Write(p []byte) (int, error)