engine

Documentation

Index

• type Executor
  • func NewExecutor(binaryPath string) *Executor
  • func (e *Executor) LoadModel(modelPath string) error
  • func (e *Executor) Shutdown() error
  • func (e *Executor) StartInference(config InferenceConfig) (<-chan string, error)
  • func (e *Executor) Stop() error
• type InferenceConfig
  • func DefaultConfig() InferenceConfig
• type ServerRequest
• type ServerResponse

Types

type Executor

type Executor struct {
	// contains filtered or unexported fields
}

func NewExecutor

func NewExecutor(binaryPath string) *Executor

func (*Executor) LoadModel

func (e *Executor) LoadModel(modelPath string) error

LoadModel starts the server without running inference

func (*Executor) Shutdown

func (e *Executor) Shutdown() error

Shutdown actually kills the server process (used when closing app)

func (*Executor) StartInference

func (e *Executor) StartInference(config InferenceConfig) (<-chan string, error)

func (*Executor) Stop

func (e *Executor) Stop() error

Stop cancels the CURRENT GENERATION but keeps the server running

type InferenceConfig

type InferenceConfig struct {
	ModelPath     string  `json:"model_path"`
	Prompt        string  `json:"prompt"`
	SystemPrompt  string  `json:"system_prompt"`
	Temperature   float64 `json:"temperature"`    // 0.0 to 2.0
	TopP          float64 `json:"top_p"`          // 0.0 to 1.0
	TopK          int     `json:"top_k"`          // 0 to 100
	RepeatPenalty float64 `json:"repeat_penalty"` // 1.0 to 2.0
	MaxTokens     int     `json:"max_tokens"`     // -1 for infinite
	Threads       int     `json:"threads"`        // number of CPU threads
}

InferenceConfig holds parameters for text generation

func DefaultConfig

func DefaultConfig() InferenceConfig

type ServerRequest

type ServerRequest struct {
	Prompt        string  `json:"prompt"`
	NPredict      int     `json:"n_predict"`
	Temperature   float64 `json:"temperature"`
	TopP          float64 `json:"top_p"`
	TopK          int     `json:"top_k"`
	RepeatPenalty float64 `json:"repeat_penalty"`
	Stream        bool    `json:"stream"`
}

type ServerResponse

type ServerResponse struct {
	Content string `json:"content"`
	Stop    bool   `json:"stop"`
}