Documentation ¶
Index ¶
- type CacheManager
- type LRUCache
- func (cache *LRUCache) BaseDir() string
- func (cache *LRUCache) EnsureFreeBytes(bytes int64)
- func (cache *LRUCache) Get(item ModelIdentifier) (Model, bool)
- func (cache *LRUCache) ListModels() []*Model
- func (cache *LRUCache) ModelPath(model Model) string
- func (cache *LRUCache) Put(item ModelIdentifier, model Model)
- type Model
- type ModelCache
- type ModelIdentifier
- type ModelProvider
- type ModelVersionStatus_State
- type TFServingController
- func (server *TFServingController) Close() error
- func (server *TFServingController) GetModelStates() ([]ModelVersionStatus_State, error)
- func (server *TFServingController) GetModelStatus(model Model) (ModelVersionStatus_State, error)
- func (server *TFServingController) ReloadConfig(models []*Model, tfServingServerModelDir string) error
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CacheManager ¶
type CacheManager struct { RestProxy *tfservingproxy.RestProxy GrpcProxy *tfservingproxy.GrpcProxy ModelProvider ModelProvider LocalCache ModelCache MaxConcurrentModels int TFServingServerModelBasePath string ServingController *TFServingController ModelFetchTimeout float32 // model fetch timeout in seconds // contains filtered or unexported fields }
func New ¶
func New( modelProvider ModelProvider, modelCache ModelCache, tfServingServerBasePath string, tfservingServerGRPCHost string, tfservingServerRESTHost string, modelFetchTimeout float32, maxConcurrentModels int, ) *CacheManager
func (*CacheManager) Close ¶
func (cache *CacheManager) Close() error
func (*CacheManager) ServeRest ¶
func (handler *CacheManager) ServeRest() func(http.ResponseWriter, *http.Request)
type LRUCache ¶
type LRUCache struct { Capacity int64 // contains filtered or unexported fields }
func NewLRUCache ¶
func (*LRUCache) EnsureFreeBytes ¶
Deletes LRU models until number of bytes are available
func (*LRUCache) Get ¶
func (cache *LRUCache) Get(item ModelIdentifier) (Model, bool)
Retrieves an item from the cache as well as a bool indicating whether the item was present or not. If the item is not present, the zero value of the type is returned.
func (*LRUCache) ListModels ¶
func (*LRUCache) Put ¶
func (cache *LRUCache) Put(item ModelIdentifier, model Model)
Adds an item to the cache (if it does not already exist)
type Model ¶
type Model struct { Identifier ModelIdentifier Path string SizeOnDisk int64 }
type ModelCache ¶
type ModelCache interface { BaseDir() string ModelPath(model Model) string Put(item ModelIdentifier, model Model) Get(item ModelIdentifier) (Model, bool) ListModels() []*Model EnsureFreeBytes(bytes int64) }
type ModelIdentifier ¶
type ModelProvider ¶
type ModelVersionStatus_State ¶
type ModelVersionStatus_State int32
TYPES FROM TENSORFLOW COPIED FOR CONVENIENCE
const ( // Default value. ModelVersionStatus_UNKNOWN ModelVersionStatus_State = 0 // The manager is tracking this servable, but has not initiated any action // pertaining to it. ModelVersionStatus_START ModelVersionStatus_State = 10 // The manager has decided to load this servable. In particular, checks // around resource availability and other aspects have passed, and the // manager is about to invoke the loader's Load() method. ModelVersionStatus_LOADING ModelVersionStatus_State = 20 // The manager has successfully loaded this servable and made it available // for serving (i.e. GetServableHandle(id) will succeed). To avoid races, // this state is not reported until *after* the servable is made // available. ModelVersionStatus_AVAILABLE ModelVersionStatus_State = 30 // The manager has decided to make this servable unavailable, and unload // it. To avoid races, this state is reported *before* the servable is // made unavailable. ModelVersionStatus_UNLOADING ModelVersionStatus_State = 40 // This servable has reached the end of its journey in the manager. Either // it loaded and ultimately unloaded successfully, or it hit an error at // some point in its lifecycle. ModelVersionStatus_END ModelVersionStatus_State = 50 )
func (*ModelVersionStatus_State) String ¶
func (state *ModelVersionStatus_State) String() string
type TFServingController ¶
type TFServingController struct {
// contains filtered or unexported fields
}
func NewTFServingController ¶
func NewTFServingController(grpcHost string, restHost string) (*TFServingController, error)
func (*TFServingController) Close ¶
func (server *TFServingController) Close() error
func (*TFServingController) GetModelStates ¶
func (server *TFServingController) GetModelStates() ([]ModelVersionStatus_State, error)
func (*TFServingController) GetModelStatus ¶
func (server *TFServingController) GetModelStatus(model Model) (ModelVersionStatus_State, error)
func (*TFServingController) ReloadConfig ¶
func (server *TFServingController) ReloadConfig(models []*Model, tfServingServerModelDir string) error
Source Files ¶
Click to show internal directories.
Click to hide internal directories.