backend

Documentation

Index

• func Finetune(config config.BackendConfig, input, prediction string) string
• func ImageGeneration(height, width, mode, step, seed int, ...) (func() error, error)
• func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, ...) (func() ([]float32, error), error)
• func ModelInference(ctx context.Context, s string, messages []schema.Message, ...) (func() (LLMResponse, error), error)
• func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option
• func ModelTTS(text, voice, language string, loader *model.ModelLoader, ...) (string, *proto.Result, error)
• func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.BackendConfig, ...) (schema.TokenizeResponse, error)
• func ModelTranscription(audio, language string, translate bool, ml *model.ModelLoader, ...) (*schema.TranscriptionResult, error)
• func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, ...) (*proto.RerankResult, error)
• func SoundGeneration(text string, duration *float32, temperature *float32, doSample *bool, ...) (string, *proto.Result, error)
• func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string) (grpc.Backend, error)
• func TokenMetrics(modelFile string, loader *model.ModelLoader, ...) (*proto.MetricsResponse, error)
• func VAD(request *schema.VADRequest, ctx context.Context, ml *model.ModelLoader, ...) (*schema.VADResponse, error)
• func VideoGeneration(height, width int32, prompt, startImage, endImage, dst string, ...) (func() error, error)
• type LLMResponse
• type TokenUsage

Functions

func Finetune

func Finetune(config config.BackendConfig, input, prediction string) string

func ImageGeneration

func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error)

func ModelEmbedding

func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() ([]float32, error), error)

func ModelInference

func ModelInference(ctx context.Context, s string, messages []schema.Message, images, videos, audios []string, loader *model.ModelLoader, c *config.BackendConfig, cl *config.BackendConfigLoader, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error)

func ModelOptions

func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option

func ModelTTS

func ModelTTS(
	text,
	voice,
	language string,
	loader *model.ModelLoader,
	appConfig *config.ApplicationConfig,
	backendConfig config.BackendConfig,
) (string, *proto.Result, error)

func ModelTokenize

func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (schema.TokenizeResponse, error)

func ModelTranscription

func ModelTranscription(audio, language string, translate bool, ml *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (*schema.TranscriptionResult, error)

func Rerank

func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, backendConfig config.BackendConfig) (*proto.RerankResult, error)

func SoundGeneration

func SoundGeneration(
	text string,
	duration *float32,
	temperature *float32,
	doSample *bool,
	sourceFile *string,
	sourceDivisor *int32,
	loader *model.ModelLoader,
	appConfig *config.ApplicationConfig,
	backendConfig config.BackendConfig,
) (string, *proto.Result, error)

func StoreBackend

func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string) (grpc.Backend, error)

func TokenMetrics

func TokenMetrics(
	modelFile string,
	loader *model.ModelLoader,
	appConfig *config.ApplicationConfig,
	backendConfig config.BackendConfig) (*proto.MetricsResponse, error)

func VAD

func VAD(request *schema.VADRequest,
	ctx context.Context,
	ml *model.ModelLoader,
	appConfig *config.ApplicationConfig,
	backendConfig config.BackendConfig) (*schema.VADResponse, error)

func VideoGeneration

func VideoGeneration(height, width int32, prompt, startImage, endImage, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error)

Types

type LLMResponse

type LLMResponse struct {
	Response    string // should this be []byte?
	Usage       TokenUsage
	AudioOutput string
}

type TokenUsage

type TokenUsage struct {
	Prompt                 int
	Completion             int
	TimingPromptProcessing float64
	TimingTokenGeneration  float64
}