types

Documentation

Index

• type ApiType
• type Delta
• type DeltaChoice
• type GoInferConf
• type InferenceParams
• type InferenceResult
• type InferenceStats
• type ModelConf
• type MsgType
• type OpenAiChatCompletion
• type OpenAiChatCompletionDeltaResponse
• type OpenAiChoice
• type OpenAiConf
• type OpenAiMessage
• type OpenAiMessages
• type OpenAiModel
• type OpenAiUsage
• type StreamedMessage
• type Task
• type TemplateInfo

Types

type ApiType

type ApiType string

const (
	Llama  ApiType = "llama"
	OpenAi ApiType = "openai"
)

type Delta

type Delta struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type DeltaChoice

type DeltaChoice struct {
	Delta        Delta  `json:"delta"`
	Index        int    `json:"index"`
	FinishReason string `json:"finish_reason,omitempty"`
}

type GoInferConf

type GoInferConf struct {
	ModelsDir  string
	TasksDir   string
	Origins    []string
	ApiKey     string
	OpenAiConf OpenAiConf
}

type InferenceParams

type InferenceParams struct {
	Stream            bool     `json:"stream,omitempty" yaml:"stream,omitempty"`
	Threads           int      `json:"threads,omitempty" yaml:"threads,omitempty"`
	NPredict          int      `json:"n_predict,omitempty" yaml:"n_predict,omitempty"`
	TopK              int      `json:"top_k,omitempty" yaml:"top_k,omitempty"`
	TopP              float32  `json:"top_p,omitempty" yaml:"top_p,omitempty"`
	Temperature       float32  `json:"temperature,omitempty" yaml:"temperature,omitempty"`
	FrequencyPenalty  float32  `json:"frequency_penalty,omitempty" yaml:"frequency_penalty,omitempty"`
	PresencePenalty   float32  `json:"presence_penalty,omitempty" yaml:"presence_penalty,omitempty"`
	RepeatPenalty     float32  `json:"repeat_penalty,omitempty" yaml:"repeat_penalty,omitempty"`
	TailFreeSamplingZ float32  `json:"tfs_z,omitempty" yaml:"tfs_z,omitempty"`
	StopPrompts       []string `json:"stop,omitempty" yaml:"stop,omitempty"`
}

type InferenceResult

type InferenceResult struct {
	Text  string         `json:"text"`
	Stats InferenceStats `json:"stats"`
}

type InferenceStats

type InferenceStats struct {
	ThinkingTime       float64 `json:"thinkingTime"`
	ThinkingTimeFormat string  `json:"thinkingTimeFormat"`
	EmitTime           float64 `json:"emitTime"`
	EmitTimeFormat     string  `json:"emitTimeFormat"`
	TotalTime          float64 `json:"totalTime"`
	TotalTimeFormat    string  `json:"totalTimeFormat"`
	TokensPerSecond    float64 `json:"tokensPerSecond"`
	TotalTokens        int     `json:"totalTokens"`
}

type ModelConf

type ModelConf struct {
	Name      string `json:"name" yaml:"name"`
	Ctx       int    `json:"ctx,omitempty" yaml:"ctx,omitempty"`
	GPULayers int    `json:"gpu_layers,omitempty" yaml:"gpu_layers,omitempty"`
}

type MsgType

type MsgType string

const (
	TokenMsgType  MsgType = "token"
	SystemMsgType MsgType = "system"
	ErrorMsgType  MsgType = "error"
)

type OpenAiChatCompletion

type OpenAiChatCompletion struct {
	ID      string         `json:"id"`
	Object  string         `json:"object"`
	Created int64          `json:"created"`
	Model   string         `json:"model"`
	Choices []OpenAiChoice `json:"choices"`
	Usage   OpenAiUsage    `json:"usage"`
}

type OpenAiChatCompletionDeltaResponse

type OpenAiChatCompletionDeltaResponse struct {
	ID      string        `json:"id"`
	Object  string        `json:"object"`
	Created int64         `json:"created"`
	Model   string        `json:"model"`
	Choices []DeltaChoice `json:"choices"`
}

type OpenAiChoice

type OpenAiChoice struct {
	Index        int           `json:"index"`
	Message      OpenAiMessage `json:"message"`
	FinishReason string        `json:"finish_reason"`
}

type OpenAiConf

type OpenAiConf struct {
	Enable   bool   `json:"enable"`
	Threads  int    `json:"threads"`
	Template string `json:"template"`
}

type OpenAiMessage

type OpenAiMessage struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type OpenAiMessages

type OpenAiMessages struct {
	Model            string          `json:"model"`
	Messages         []OpenAiMessage `json:"messages"`
	Temperature      float64         `json:"temperature,omitempty"`
	TopP             float64         `json:"top_p,omitempty"`
	N                int             `json:"n,omitempty"`
	Stream           bool            `json:"stream,omitempty"`
	Stop             interface{}     `json:"stop,omitempty"`
	MaxTokens        int             `json:"max_tokens,omitempty"`
	PresencePenalty  float64         `json:"presence_penalty,omitempty"`
	FrequencyPenalty float64         `json:"frequency_penalty,omitempty"`
	LogitBias        map[string]int  `json:"logit_bias,omitempty"`
}

type OpenAiModel

type OpenAiModel struct {
	ID      string `json:"id"`
	Object  string `json:"object"`
	Created int64  `json:"created"`
	OwnedBy string `json:"owned_by"`
}

type OpenAiUsage

type OpenAiUsage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}

type StreamedMessage

type StreamedMessage struct {
	Content string                 `json:"content"`
	Num     int                    `json:"num"`
	MsgType MsgType                `json:"msg_type"`
	Data    map[string]interface{} `json:"data,omitempty"`
}

type Task

type Task struct {
	Name        string          `json:"name" yaml:"name"`
	Template    string          `json:"template" yaml:"template"`
	ModelConf   ModelConf       `json:"modelConf" yaml:"modelConf"`
	InferParams InferenceParams `json:"inferParams" yaml:"inferParams"`
}

type TemplateInfo

type TemplateInfo struct {
	Name string `json:"name" yaml:"name"`
	Ctx  int    `json:"ctx" yaml:"ctx"`
}