modelproxy

Documentation

Index

• Variables
• type Handler
  • func NewHandler(modelClient ModelClient, loadBalancer LoadBalancer, maxRetries int, ...) *Handler
  • func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request)
• type LoadBalancer
• type ModelClient

Variables

var AdditionalProxyRewrite = func(*httputil.ProxyRequest) {}

AdditionalProxyRewrite is an injection point for modifying proxy requests. Used in tests.

var ErrRetry = errors.New("retry")

Types

type Handler

type Handler struct {
	// contains filtered or unexported fields
}

Handler serves http requests for end-clients. It is also responsible for triggering scale-from-zero.

func NewHandler

func NewHandler(
	modelClient ModelClient,
	loadBalancer LoadBalancer,
	maxRetries int,
	retryCodes map[int]struct{},
) *Handler

func (*Handler) ServeHTTP

func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request)

type LoadBalancer

type LoadBalancer interface {
	AwaitBestAddress(ctx context.Context, req *apiutils.Request) (string, func(), error)
}

type ModelClient

type ModelClient interface {
	LookupModel(ctx context.Context, model, adapter string, selectors []string) (*v1.Model, error)
	ScaleAtLeastOneReplica(ctx context.Context, model string) error
}