veloxquant

package module

Go to main page

Versions in this module

v0

v0.5.0

Sep 8, 2026

Changes in this version

+ var ErrHuggingFaceHubUnavailable = models.ErrHuggingFaceHubUnavailable

+ type BenchmarkInput struct

+ MaxTokens int

+ Model string

+ OptimizedMethod string

+ type BenchmarkResult struct

+ Chip string

+ DefaultMethodResidentBytes *uint64

+ Method string

+ Model string

+ OptimizedMethodUsed string

+ OptimizedResidentBytes *uint64

+ TimeToFirstTokenMs float64

+ TokensPerSecond float64

+ UnifiedMemoryBytes uint64

+ func Benchmark(ctx context.Context, client *Client, input BenchmarkInput) (BenchmarkResult, error)

+ func (r BenchmarkResult) ToMarkdown() string

type ChatRequest

+ Tools []ToolDefinition

type ChatResponse

+ FinishReason string

+ ToolCalls []ToolCall

+ type DeleteResult = models.DeleteResult

type Message

+ ToolCallID string

+ ToolCalls []ToolCall

type ModelsService

+ func (m *ModelsService) Delete(ctx context.Context, python string, modelID string) (DeleteResult, error)

+ func (m *ModelsService) Pull(ctx context.Context, python string, modelID string) (PullResult, error)

+ type PullResult = models.PullResult

+ type ToolCall = openai.ToolCall

+ type ToolCallFunction = openai.ToolCallFunction

+ type ToolDefinition = openai.ToolDefinition

+ type ToolFunction = openai.ToolFunction

v0.4.0

Sep 4, 2026

Changes in this version

type ChatRequest

+ ResponseFormat *ResponseFormat

type Client

+ func (c *Client) Embed(ctx context.Context, req EmbedRequest) (EmbedResponse, error)

+ func (c *Client) NewConversation(model string, system string) *Conversation

+ type Conversation struct

+ func (conv *Conversation) History() []Message

+ func (conv *Conversation) Send(ctx context.Context, prompt string) (ChatResponse, error)

+ func (conv *Conversation) SendStream(ctx context.Context, prompt string) (*ConversationStream, error)

+ type ConversationStream struct

+ func (cs *ConversationStream) Chunk() ChatChunk

+ func (cs *ConversationStream) Close() error

+ func (cs *ConversationStream) Err() error

+ func (cs *ConversationStream) Next() bool

+ type EmbedRequest struct

+ Input any

+ Model string

+ type EmbedResponse struct

+ Data []Embedding

+ Model string

+ Usage Usage

+ type Embedding struct

+ Index int

+ Vector []float64

+ type JSONSchemaFormat = openai.JSONSchema

+ type LocalModelInfo struct

+ LastModified *time.Time

+ Name string

+ Path string

+ SizeBytes uint64

type ModelsService

+ func (m *ModelsService) Local(ctx context.Context) ([]LocalModelInfo, error)

+ type ResponseFormat = openai.ResponseFormat

+ func JSONMode() *ResponseFormat

+ func JSONSchema(name string, schema any, strict bool) *ResponseFormat

type Session

+ func (s *Session) Conversation(system string) *Conversation

v0.3.0

Sep 3, 2026

Changes in this version

type ChatStream

+ func (s *ChatStream) Metrics() InferenceMetrics

type ModelsService

+ func (m *ModelsService) RecommendScored(ctx context.Context, req ModelRecommendationRequest) ([]models.Scored, error)

+ type MonitorConfig struct

+ type MonitorOption func(*MonitorConfig)

+ func WithMonitorInterval(interval time.Duration) MonitorOption

v0.1.0

Sep 2, 2026

Changes in this version

+ const FP16

+ const FP8

+ const Int4

+ const Int8

+ var ErrInsufficientMemory = errors.New("insufficient memory")

+ var ErrInvalidConfig = errors.New("invalid configuration")

+ var ErrModelNotFound = errors.New("model not found")

+ var ErrRuntimeUnavailable = errors.New("veloxquant runtime unavailable")

+ var ErrUnsupportedPlatform = errors.New("unsupported platform")

+ func FormatBytes(b uint64) string

+ type AutoPilotConfig struct

+ ContextLength int

+ Model string

+ Task string

+ type AutoPilotPlan struct

+ CompressionBits int

+ ContextLength int

+ EstimatedMemoryBytes uint64

+ Hardware SystemInfo

+ Profile optimize.Profile

+ Reason string

+ SafetyMarginBytes uint64

+ SelectedModel string

+ type ChatChunk struct

+ Done bool

+ Text string

+ type ChatRequest struct

+ MaxTokens int

+ Messages []Message

+ Model string

+ Stream bool

+ Temperature float64

+ type ChatResponse struct

+ ID string

+ Metrics InferenceMetrics

+ Model string

+ Text string

+ Usage Usage

+ type ChatStream struct

+ func (s *ChatStream) Chunk() ChatChunk

+ func (s *ChatStream) Close() error

+ func (s *ChatStream) Err() error

+ func (s *ChatStream) Next() bool

+ type Client struct

+ Memory *MemoryService

+ Models *ModelsService

+ Optimize *OptimizeService

+ Runtime *RuntimeService

+ System *SystemService

+ func NewClient(opts ...Option) (*Client, error)

+ func (c *Client) AutoPilot(ctx context.Context, cfg AutoPilotConfig) (*Session, error)

+ func (c *Client) Chat(ctx context.Context, req ChatRequest) (ChatResponse, error)

+ func (c *Client) ChatStream(ctx context.Context, req ChatRequest) (*ChatStream, error)

+ func (c *Client) Monitor() *monitor.Monitor

+ type InferenceMetrics struct

+ TimeToFirstToken time.Duration

+ TokensPerSecond float64

+ TotalDuration time.Duration

+ type MemoryEstimate struct

+ KVCacheMemoryBytes uint64

+ ModelMemoryBytes uint64

+ OptimizedKVBytes uint64

+ OptimizedTotalBytes uint64

+ RecommendedStrategy string

+ RuntimeOverheadBytes uint64

+ SavedBytes uint64

+ SavedPercent float64

+ TotalMemoryBytes uint64

+ type MemoryRequest struct

+ ContextLength int

+ Model ModelArchitecture

+ Precision Precision

+ type MemoryService struct

+ func (m *MemoryService) Estimate(ctx context.Context, req MemoryRequest) (MemoryEstimate, error)

+ type Message struct

+ Content string

+ Role string

+ type ModelArchitecture = memory.Architecture

+ type ModelRecommendationRequest struct

+ AvailableMemoryBytes uint64

+ ContextLength int

+ Task string

+ type ModelsService struct

+ func (m *ModelsService) List() []models.Info

+ func (m *ModelsService) Recommend(ctx context.Context, req ModelRecommendationRequest) ([]models.Info, error)

+ type OptimizationRecommendation struct

+ CompressionBits int

+ CompressionMethod string

+ ContextLength int

+ EstimatedMemoryAfter uint64

+ EstimatedMemoryBefore uint64

+ Profile optimize.Profile

+ Reason string

+ type OptimizationRequest struct

+ Architecture ModelArchitecture

+ ContextLength int

+ Model string

+ type OptimizeService struct

+ func (o *OptimizeService) Recommend(ctx context.Context, req OptimizationRequest) (OptimizationRecommendation, error)

+ type Option func(*config)

+ func WithAutoDetect() Option

+ func WithHTTPTimeout(timeout time.Duration) Option

+ func WithOpenAICompatibleRuntime(baseURL string) Option

+ func WithProfile(profile string) Option

+ func WithRuntimeURL(url string) Option

+ type Precision = memory.Precision

+ type RuntimeService struct

+ func (r *RuntimeService) Health(ctx context.Context) (RuntimeStatus, error)

+ type RuntimeStatus struct

+ Engine string

+ Healthy bool

+ Version string

+ type Session struct

+ func (s *Session) Chat(ctx context.Context, prompt string) (ChatResponse, error)

+ func (s *Session) Plan() AutoPilotPlan

+ type SystemInfo struct

+ AppleSilicon bool

+ Architecture string

+ AvailableMemory uint64

+ CPUModel string

+ Platform string

+ RecommendedProfile string

+ TotalMemory uint64

+ type SystemService struct

+ func (s *SystemService) Info(ctx context.Context) (SystemInfo, error)

+ type Usage struct

+ CompletionTokens int

+ PromptTokens int

+ TotalTokens int

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL