vllm

Documentation

Index

• func ChatTemplate(modelName string) (string, error)
• func IsAWQQuantizedModel(modelName string) bool
• func ModelFilePath(modelDir, modelName string, format mv1.ModelFormat) (string, error)
• func PreferredModelFormat(resp *mv1.GetBaseModelPathResponse) (mv1.ModelFormat, error)
• type Manager
  • func New(modelDir string, s3Client s3Client) *Manager
  • func (m *Manager) CreateNewModelOfGGUF(modelName string, spec *ollama.ModelSpec) error
  • func (m *Manager) DownloadAndCreateNewModel(ctx context.Context, modelName string, resp *mv1.GetBaseModelPathResponse) error
  • func (m *Manager) UpdateModelTemplateToLatest(modelName string) error

Functions

func ChatTemplate

func ChatTemplate(modelName string) (string, error)

ChatTemplate returns the chat template for the given model.

func IsAWQQuantizedModel

func IsAWQQuantizedModel(modelName string) bool

IsAWQQuantizedModel returns true if the model name is an AWQ quantized model.

func ModelFilePath

func ModelFilePath(modelDir, modelName string, format mv1.ModelFormat) (string, error)

ModelFilePath returns the file path of the model.

func PreferredModelFormat

func PreferredModelFormat(resp *mv1.GetBaseModelPathResponse) (mv1.ModelFormat, error)

PreferredModelFormat returns the preferred model format.

Types

type Manager

type Manager struct {
	// contains filtered or unexported fields
}

Manager manages the Ollama service.

TODO(kenji): Refactor this class once we completely switch to the one-odel-per-pod implementation where inference-manager-engine doesn't directly run vLLM or Ollama.

func New

func New(modelDir string, s3Client s3Client) *Manager

New returns a new Manager.

func (*Manager) CreateNewModelOfGGUF

func (m *Manager) CreateNewModelOfGGUF(modelName string, spec *ollama.ModelSpec) error

CreateNewModelOfGGUF creates a new model with the given name and spec that uses a GGUF model file.

func (*Manager) DownloadAndCreateNewModel

func (m *Manager) DownloadAndCreateNewModel(ctx context.Context, modelName string, resp *mv1.GetBaseModelPathResponse) error

DownloadAndCreateNewModel downloads the model from the given path and creates a new model.

func (*Manager) UpdateModelTemplateToLatest

func (m *Manager) UpdateModelTemplateToLatest(modelName string) error

UpdateModelTemplateToLatest updates the model template to the latest.