models

Documentation

Index

• type ChatRole
• type Choice
• type Delta
• type Message
• type Model
  • func GetInfo(uuid string, host string, port int16, useSSL bool, skipVerify bool) (*Model, error)
• type Provider
• type TgiParameters
• type TgiRequestBody
• type UniversalRequest
• type UniversalResponse
  • func DoGenerate(uReq *UniversalRequest, model *db.LLMModel, uRespChan chan string) *UniversalResponse
• type Usage

Types

type ChatRole

type ChatRole string

const (
	SYSTEM_ROLE    ChatRole = "system"
	ASSISTANT_ROLE ChatRole = "assistant"
	USER_ROLE      ChatRole = "user"
	FUNCTION_ROLE  ChatRole = "function"
)

type Choice

type Choice struct {
	Index        int      `json:"index"`
	Message      *Message `json:"message,omitempty"` //this actually whats returned in case stream: false
	Delta        *Delta   `json:"delta,omitempty"`   //this actually whats returned in case its stream: true
	FinishReason *string  `json:"finish_reason"`
}

type Delta

type Delta struct {
	Content *string `json:"content,omitempty"`
}

****************** Universal Generate/Completion Response ***************************

type Message

type Message struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type Model

type Model struct {
	ModelID               string    `json:"model_id"`
	ModelSha              string    `json:"model_sha"`
	ModelDtype            string    `json:"model_dtype"`
	ModelDeviceType       string    `json:"model_device_type"`
	ModelPipelineTag      string    `json:"model_pipeline_tag"`
	MaxConcurrentRequests int       `json:"max_concurrent_requests"`
	MaxBestOf             int       `json:"max_best_of"`
	MaxStopSequences      int       `json:"max_stop_sequences"`
	MaxInputLength        int       `json:"max_input_length"`
	MaxTotalTokens        int       `json:"max_total_tokens"`
	WaitingServedRatio    float64   `json:"waiting_served_ratio"`
	MaxBatchTotalTokens   int       `json:"max_batch_total_tokens"`
	MaxWaitingTokens      int       `json:"max_waiting_tokens"`
	ValidationWorkers     int       `json:"validation_workers"`
	Version               string    `json:"version"`
	DBModelUUID           string    `json:"-"`
	LastChecked           time.Time `json:"-"`
	IsActive              bool      `json:"-"`
	Host                  string    `json:"-"`

	UseSSL            bool `json:"-"`
	DontSkipVerifySSL bool `json:"-"`
	// contains filtered or unexported fields
}

func GetInfo

func GetInfo(uuid string, host string, port int16, useSSL bool, skipVerify bool) (*Model, error)

type Provider

type Provider interface {
}

type TgiParameters

type TgiParameters struct {
	MaxNewTokens      int      `json:"max_new_tokens"`     //required
	Temperature       float64  `json:"temperature"`        //required
	TopK              int      `json:"top_k"`              //required
	TopP              float64  `json:"top_p"`              //required
	Stop              []string `json:"stop"`               //required
	RepetitionPenalty float64  `json:"repetition_penalty"` //required
	// BestOf              int      `json:"best_of"`
	// DecoderInputDetails bool    `json:"decoder_input_details"`
	Details        bool    `json:"details"`
	DoSample       bool    `json:"do_sample"`
	ReturnFullText bool    `json:"return_full_text"`
	Seed           *int    `json:"seed"`
	Truncate       *int    `json:"truncate"`
	TypicalP       float64 `json:"typical_p"`
	Watermark      bool    `json:"watermark"`
}

type TgiRequestBody

type TgiRequestBody struct {
	Inputs     string        `json:"inputs"`
	Parameters TgiParameters `json:"parameters"`
}

type UniversalRequest

type UniversalRequest struct {
	Messages         []Message         `json:"messages"`
	Stream           bool              `json:"stream"`
	Model            string            `json:"model"`
	Temperature      float64           `json:"temperature"`
	TopP             float64           `json:"top_p"`
	N                int               `json:"n"`
	Stop             string            `json:"stop"`
	PresencePenalty  float64           `json:"presence_penalty"`
	FrequencyPenalty float64           `json:"frequency_penalty"`
	LogitBias        map[string]string `json:"logit_bias"`
}

type UniversalResponse

type UniversalResponse struct {
	ID      string   `json:"id"`
	Object  string   `json:"object"`
	Created int      `json:"created"`
	Model   string   `json:"model"`
	Choices []Choice `json:"choices"`
	Usage   *Usage   `json:"usage,omitempty"`
}

func DoGenerate

func DoGenerate(uReq *UniversalRequest, model *db.LLMModel, uRespChan chan string) *UniversalResponse

type Usage

type Usage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens"`
	TotalTokens      int `json:"total_tokens"`
}