cachemanager

Documentation

Index

• type CacheManager
  • func New(modelProvider ModelProvider, modelCache ModelCache, ...) *CacheManager
  • func (handler *CacheManager) ServeRest() func(http.ResponseWriter, *http.Request)
• type LRUCache
  • func NewLRUCache(dir string, capacityInBytes int64) LRUCache
  • func (cache *LRUCache) BaseDir() string
  • func (cache *LRUCache) EnsureFreeBytes(bytes int64)
  • func (cache *LRUCache) Get(item ModelIdentifier) (Model, bool)
  • func (cache *LRUCache) ListModels() []*Model
  • func (cache *LRUCache) ModelPath(model Model) string
  • func (cache *LRUCache) Put(item ModelIdentifier, model Model)
• type Model
• type ModelCache
• type ModelIdentifier
• type ModelProvider
• type ModelVersionStatus_State
  • func (state *ModelVersionStatus_State) String() string
• type TFServingController
  • func (server *TFServingController) GetModelStatus(model Model) (ModelVersionStatus_State, error)
  • func (server *TFServingController) ReloadConfig(models []*Model, tfServingServerModelDir string) error

Types

type CacheManager

type CacheManager struct {
	RestProxy *tfservingproxy.RestProxy

	ModelProvider                ModelProvider
	LocalCache                   ModelCache
	TFServingServerModelBasePath string
	ServingController            TFServingController
	ModelFetchTimeout            float32 // model fetch timeout in seconds
	// contains filtered or unexported fields
}

func New

func New(
	modelProvider ModelProvider,
	modelCache ModelCache,
	tfServingServerBasePath string,
	tfservingServerGRPCHost string,
	tfservingServerRESTHost string,
	modelFetchTimeout float32,
) *CacheManager

func (*CacheManager) ServeRest

func (handler *CacheManager) ServeRest() func(http.ResponseWriter, *http.Request)

type LRUCache

type LRUCache struct {
	Capacity int64
	// contains filtered or unexported fields
}

func NewLRUCache

func NewLRUCache(dir string, capacityInBytes int64) LRUCache

func (*LRUCache) BaseDir

func (cache *LRUCache) BaseDir() string

func (*LRUCache) EnsureFreeBytes

func (cache *LRUCache) EnsureFreeBytes(bytes int64)

Deletes LRU models until number of bytes are available

func (*LRUCache) Get

func (cache *LRUCache) Get(item ModelIdentifier) (Model, bool)

Retrieves an item from the cache as well as a bool indicating whether the item was present or not. If the item is not present, the zero value of the type is returned.

func (*LRUCache) ListModels

func (cache *LRUCache) ListModels() []*Model

func (*LRUCache) ModelPath

func (cache *LRUCache) ModelPath(model Model) string

func (*LRUCache) Put

func (cache *LRUCache) Put(item ModelIdentifier, model Model)

Adds an item to the cache (if it does not already exist)

type Model

type Model struct {
	Identifier ModelIdentifier
	Path       string
	SizeOnDisk int64
}

type ModelCache

type ModelCache interface {
	BaseDir() string
	ModelPath(model Model) string
	Put(item ModelIdentifier, model Model)
	Get(item ModelIdentifier) (Model, bool)
	ListModels() []*Model
	EnsureFreeBytes(bytes int64)
}

type ModelIdentifier

type ModelIdentifier struct {
	ModelName string
	Version   int64
}

type ModelProvider

type ModelProvider interface {
	LoadModel(modelName string, modelVersion int64, destinationDir string) (Model, error)
	ModelSize(modelName string, modelVersion int64) (int64, error)
}

type ModelVersionStatus_State

type ModelVersionStatus_State int32

TYPES FROM TENSORFLOW COPIED FOR CONVENIENCE

const (
	// Default value.
	ModelVersionStatus_UNKNOWN ModelVersionStatus_State = 0
	// The manager is tracking this servable, but has not initiated any action
	// pertaining to it.
	ModelVersionStatus_START ModelVersionStatus_State = 10
	// The manager has decided to load this servable. In particular, checks
	// around resource availability and other aspects have passed, and the
	// manager is about to invoke the loader's Load() method.
	ModelVersionStatus_LOADING ModelVersionStatus_State = 20
	// The manager has successfully loaded this servable and made it available
	// for serving (i.e. GetServableHandle(id) will succeed). To avoid races,
	// this state is not reported until *after* the servable is made
	// available.
	ModelVersionStatus_AVAILABLE ModelVersionStatus_State = 30
	// The manager has decided to make this servable unavailable, and unload
	// it. To avoid races, this state is reported *before* the servable is
	// made unavailable.
	ModelVersionStatus_UNLOADING ModelVersionStatus_State = 40
	// This servable has reached the end of its journey in the manager. Either
	// it loaded and ultimately unloaded successfully, or it hit an error at
	// some point in its lifecycle.
	ModelVersionStatus_END ModelVersionStatus_State = 50
)

func (*ModelVersionStatus_State) String

func (state *ModelVersionStatus_State) String() string

type TFServingController

type TFServingController struct {
	// contains filtered or unexported fields
}

func (*TFServingController) GetModelStatus

func (server *TFServingController) GetModelStatus(model Model) (ModelVersionStatus_State, error)

func (*TFServingController) ReloadConfig

func (server *TFServingController) ReloadConfig(models []*Model, tfServingServerModelDir string) error