inference

package
v2.14.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 8, 2023 License: BSD-3-Clause Imports: 5 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var File_tensorflow_core_grappler_optimizers_inference_batch_op_rewriter_proto protoreflect.FileDescriptor

Functions

This section is empty.

Types

type BatchOpRewriteConfig 

type BatchOpRewriteConfig struct {
	EnableAdaptiveSharedBatchingThreadPool bool `` /* 184-byte string literal not displayed */
	// DEPRECATED. Use the adaptive_batch_scheduler_option field in batch_options.
	//
	// Keyed by model name, meaning all batch-ops in one saved model would use the
	// same adaptive-batch-scheduler option.
	//
	// Deprecated: Marked as deprecated in tensorflow/core/grappler/optimizers/inference/batch_op_rewriter.proto.
	ModelSchedulerOptions map[string]*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption `` /* 214-byte string literal not displayed */
	// The options for overriding BatchFunction op in specific models.
	//
	// When these BatchOptions are set for a given model, they will override the
	// models original batching options.
	//
	// This can be used to experiment with new batching options without updating
	// the original saved model. This is NOT intended as a long term solution for
	// updating batching options. Once the correct batching options have been
	// identified using this, the saved model should be updated to include them.
	BatchOptions map[string]*BatchOpRewriteConfig_BatchOptions `` /* 185-byte string literal not displayed */
	// contains filtered or unexported fields
}

Config for the batch op rewriter. This should be serialized and set a param in RewriterConfig with key kBatchOpRewriteParamKey.

func (*BatchOpRewriteConfig) Descriptor deprecated 

func (*BatchOpRewriteConfig) Descriptor() ([]byte, []int)

Deprecated: Use BatchOpRewriteConfig.ProtoReflect.Descriptor instead.

func (*BatchOpRewriteConfig) GetBatchOptions added in v2.13.0 

func (x *BatchOpRewriteConfig) GetBatchOptions() map[string]*BatchOpRewriteConfig_BatchOptions

func (*BatchOpRewriteConfig) GetEnableAdaptiveSharedBatchingThreadPool 

func (x *BatchOpRewriteConfig) GetEnableAdaptiveSharedBatchingThreadPool() bool

func (*BatchOpRewriteConfig) GetModelSchedulerOptions deprecated 

func (x *BatchOpRewriteConfig) GetModelSchedulerOptions() map[string]*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption

Deprecated: Marked as deprecated in tensorflow/core/grappler/optimizers/inference/batch_op_rewriter.proto.

func (*BatchOpRewriteConfig) ProtoMessage 

func (*BatchOpRewriteConfig) ProtoMessage()

func (*BatchOpRewriteConfig) ProtoReflect 

func (x *BatchOpRewriteConfig) ProtoReflect() protoreflect.Message

func (*BatchOpRewriteConfig) Reset 

func (x *BatchOpRewriteConfig) Reset()

func (*BatchOpRewriteConfig) String 

func (x *BatchOpRewriteConfig) String() string

type BatchOpRewriteConfig_AdaptiveBatchSchedulerOption 

type BatchOpRewriteConfig_AdaptiveBatchSchedulerOption struct {
	MinInflightBatchesLimit     *wrapperspb.UInt32Value `` /* 134-byte string literal not displayed */
	InitialInflightBatchesLimit *wrapperspb.UInt32Value `` /* 146-byte string literal not displayed */
	MaxInflightBatchesLimit     *wrapperspb.UInt32Value `` /* 134-byte string literal not displayed */
	// You can use QPS as a reference to decide how quickly to react to workload
	// changes.
	BatchesToAverageOver *wrapperspb.UInt32Value `protobuf:"bytes,4,opt,name=batches_to_average_over,json=batchesToAverageOver,proto3" json:"batches_to_average_over,omitempty"`
	// If set, specifies `full_batch_scheduling_boost_micros` of
	// `AdaptiveSharedBatchScheduler::Options`. A full batch is scheduled
	// before an older, nearly empty batch as long as the age gap is smaller
	// than full_batch_scheduling_boost_micros. The optimal value for this
	// parameter should be of order the batch processing latency, but must be
	// chosen carefully, as too large a value will harm tail latency.
	FullBatchSchedulingBoostMicros *wrapperspb.Int64Value `` /* 157-byte string literal not displayed */
	// contains filtered or unexported fields
}

The options for tensorflow::serving::AdaptiveSharedBatchScheduler. See AdaptiveSharedBatchScheduler::Options for meaning of each field.

NOTE: Leave this unset to pick up default settings which should work for most scenarios.

Example scenarios when tuning helps: * Latency sensitive

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Descriptor deprecated 

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Descriptor() ([]byte, []int)

Deprecated: Use BatchOpRewriteConfig_AdaptiveBatchSchedulerOption.ProtoReflect.Descriptor instead.

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetBatchesToAverageOver 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetBatchesToAverageOver() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetFullBatchSchedulingBoostMicros added in v2.13.0 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetFullBatchSchedulingBoostMicros() *wrapperspb.Int64Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetInitialInflightBatchesLimit 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetInitialInflightBatchesLimit() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMaxInflightBatchesLimit 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMaxInflightBatchesLimit() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMinInflightBatchesLimit 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) GetMinInflightBatchesLimit() *wrapperspb.UInt32Value

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoMessage 

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoMessage()

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoReflect 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) ProtoReflect() protoreflect.Message

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Reset 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) Reset()

func (*BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) String 

func (x *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption) String() string

type BatchOpRewriteConfig_BatchOptions added in v2.13.0 

type BatchOpRewriteConfig_BatchOptions struct {

	// Number of scheduling threads for processing batches of work. Determines
	// the number of batches processed in parallel.
	//
	// If enable_adaptive_shared_batching_thread_pool is enabled, this value
	// can only be set to 0.
	NumBatchThreads *int32 `protobuf:"varint,1,opt,name=num_batch_threads,json=numBatchThreads,proto3,oneof" json:"num_batch_threads,omitempty"`
	// The maximum allowed batch size. Can be larger than allowed_batch_sizes to
	// utilize large batch splitting.
	MaxBatchSize *int32 `protobuf:"varint,2,opt,name=max_batch_size,json=maxBatchSize,proto3,oneof" json:"max_batch_size,omitempty"`
	// Maximum number of microseconds to wait before outputting an incomplete
	// batch.
	BatchTimeoutMicros *int32 `protobuf:"varint,3,opt,name=batch_timeout_micros,json=batchTimeoutMicros,proto3,oneof" json:"batch_timeout_micros,omitempty"`
	// List of allowed batch sizes. The batching op will pad batches up to one
	// of those sizes. The entries must increase monotonically, and the final
	// entry must be lower or equal to max_batch_size.
	AllowedBatchSizes []int32 `protobuf:"varint,4,rep,packed,name=allowed_batch_sizes,json=allowedBatchSizes,proto3" json:"allowed_batch_sizes,omitempty"`
	// Maximum number of batches enqueued for processing before requests are
	// failed fast.
	MaxEnqueuedBatches *int32 `protobuf:"varint,5,opt,name=max_enqueued_batches,json=maxEnqueuedBatches,proto3,oneof" json:"max_enqueued_batches,omitempty"`
	// If set, disables large batch splitting which is an efficiency improvement
	// on batching to reduce padding inefficiency.
	DisableLargeBatchSplitting *bool `` /* 150-byte string literal not displayed */
	// Adaptive batch scheduler options.
	AdaptiveBatchSchedulerOption *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption `` /* 155-byte string literal not displayed */
	// contains filtered or unexported fields
}

func (*BatchOpRewriteConfig_BatchOptions) Descriptor deprecated added in v2.13.0 

func (*BatchOpRewriteConfig_BatchOptions) Descriptor() ([]byte, []int)

Deprecated: Use BatchOpRewriteConfig_BatchOptions.ProtoReflect.Descriptor instead.

func (*BatchOpRewriteConfig_BatchOptions) GetAdaptiveBatchSchedulerOption added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) GetAdaptiveBatchSchedulerOption() *BatchOpRewriteConfig_AdaptiveBatchSchedulerOption

func (*BatchOpRewriteConfig_BatchOptions) GetAllowedBatchSizes added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) GetAllowedBatchSizes() []int32

func (*BatchOpRewriteConfig_BatchOptions) GetBatchTimeoutMicros added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) GetBatchTimeoutMicros() int32

func (*BatchOpRewriteConfig_BatchOptions) GetDisableLargeBatchSplitting added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) GetDisableLargeBatchSplitting() bool

func (*BatchOpRewriteConfig_BatchOptions) GetMaxBatchSize added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) GetMaxBatchSize() int32

func (*BatchOpRewriteConfig_BatchOptions) GetMaxEnqueuedBatches added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) GetMaxEnqueuedBatches() int32

func (*BatchOpRewriteConfig_BatchOptions) GetNumBatchThreads added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) GetNumBatchThreads() int32

func (*BatchOpRewriteConfig_BatchOptions) ProtoMessage added in v2.13.0 

func (*BatchOpRewriteConfig_BatchOptions) ProtoMessage()

func (*BatchOpRewriteConfig_BatchOptions) ProtoReflect added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) ProtoReflect() protoreflect.Message

func (*BatchOpRewriteConfig_BatchOptions) Reset added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) Reset()

func (*BatchOpRewriteConfig_BatchOptions) String added in v2.13.0 

func (x *BatchOpRewriteConfig_BatchOptions) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.