metrics

Documentation

Index

• Constants
• Variables
• func CleanQueryCoordMetricsWithCollectionID(collectionID int64)
• func CleanupDataCoordWithCollectionID(collectionID int64)
• func CleanupDataNodeCollectionMetrics(nodeID int64, collectionID int64, channel string)
• func CleanupProxyCollectionMetrics(nodeID int64, dbName string, collection string)
• func CleanupProxyDBMetrics(nodeID int64, dbName string)
• func CleanupQueryNodeCollectionMetrics(nodeID int64, collectionID int64)
• func CleanupRootCoordDBMetrics(dbName string)
• func GetRegisterer() prometheus.Registerer
• func NewGRPCSizeStatsHandler() *grpcSizeStatsHandler
• func RecordRestfulMetrics(ctx context.Context, outputLength int64, observeOutbound bool)
• func Register(r prometheus.Registerer)
• func RegisterCGOMetrics(registry *prometheus.Registry)
• func RegisterDataCoord(registry *prometheus.Registry)
• func RegisterDataNode(registry *prometheus.Registry)
• func RegisterMQType(mq string)
• func RegisterMetaMetrics(registry *prometheus.Registry)
• func RegisterMetaType(meta string)
• func RegisterMixCoord(registry *prometheus.Registry)
• func RegisterMsgStreamMetrics(registry *prometheus.Registry)
• func RegisterProxy(registry *prometheus.Registry)
• func RegisterQueryCoord(registry *prometheus.Registry)
• func RegisterQueryNode(registry *prometheus.Registry)
• func RegisterStorageMetrics(registry *prometheus.Registry)
• func RegisterStreamingNode(registry *prometheus.Registry)
• func RegisterStreamingServiceClient(registry *prometheus.Registry)
• func WrapRestfulContext(ctx context.Context, inputLength int64) context.Context
• type RPCStats
  • func GetStats(ctx context.Context) *RPCStats
  • func (s *RPCStats) SetCollectionName(collName string) *RPCStats
  • func (s *RPCStats) SetDatabaseName(collName string) *RPCStats
  • func (s *RPCStats) SetInboundLabel(label string) *RPCStats
  • func (s *RPCStats) SetNodeID(nodeID int64) *RPCStats

Constants

const (
	InsertFileLabel = "insert_file"
	DeleteFileLabel = "delete_file"
	StatFileLabel   = "stat_file"
	IndexFileLabel  = "index_file"
)

const (
	MetaGetLabel    = "get"
	MetaPutLabel    = "put"
	MetaRemoveLabel = "remove"
	MetaTxnLabel    = "txn"
)

const (
	AbandonLabel  = "abandon"
	SuccessLabel  = "success"
	FailLabel     = "fail"
	CancelLabel   = "cancel"
	TotalLabel    = "total"
	RetryLabel    = "retry"
	RejectedLabel = "rejected"

	HybridSearchLabel = "hybrid_search"

	InsertLabel    = "insert"
	DeleteLabel    = "delete"
	UpsertLabel    = "upsert"
	SearchLabel    = "search"
	QueryLabel     = "query"
	CacheHitLabel  = "hit"
	CacheMissLabel = "miss"
	TimetickLabel  = "timetick"
	AllLabel       = "all"

	UnissuedIndexTaskLabel   = "unissued"
	InProgressIndexTaskLabel = "in-progress"
	FinishedIndexTaskLabel   = "finished"
	FailedIndexTaskLabel     = "failed"
	RecycledIndexTaskLabel   = "recycled"

	// Note: below must matchcommonpb.SegmentState_name fields.
	SealedSegmentLabel   = "Sealed"
	GrowingSegmentLabel  = "Growing"
	FlushedSegmentLabel  = "Flushed"
	FlushingSegmentLabel = "Flushing"
	DroppedSegmentLabel  = "Dropped"

	StreamingDataSourceLabel  = "streaming"
	BulkinsertDataSourceLabel = "bulkinsert"
	CompactionDataSourceLabel = "compaction"

	Leader     = "OnLeader"
	FromLeader = "FromLeader"

	HookBefore = "before"
	HookAfter  = "after"
	HookMock   = "mock"

	ReduceSegments = "segments"
	ReduceShards   = "shards"

	BatchReduce  = "batch_reduce"
	StreamReduce = "stream_reduce"

	Pending   = "pending"
	Executing = "executing"
	Done      = "done"

	ImportStagePending      = "pending"
	ImportStagePreImport    = "preimport"
	ImportStageImport       = "import"
	ImportStageStats        = "stats"
	ImportStageBuildIndex   = "build_index"
	ImportStageWaitL0Import = "wait_l0_import"

	ResourceGroupLabelName = "rg"

	// entities label
	LoadedLabel         = "loaded"
	NumEntitiesAllLabel = "all"

	TaskTypeLabel  = "task_type"
	TaskStateLabel = "task_state"
)

const (
	SendMsgLabel    = "produce"
	ReceiveMsgLabel = "consume" // not used

	CreateProducerLabel = "create_producer"
	CreateConsumerLabel = "create_consumer"
)

const (
	DataGetLabel    = "get"
	DataPutLabel    = "put"
	DataRemoveLabel = "remove"
	DataWalkLabel   = "walk"
	DataStatLabel   = "stat"
)

const (
	SegmentGrowTaskLabel   = "segment_grow"
	SegmentReduceTaskLabel = "segment_reduce"
	SegmentMoveTaskLabel   = "segment_move"
	SegmentUpdateTaskLabel = "segment_update"

	ChannelGrowTaskLabel   = "channel_grow"
	ChannelReduceTaskLabel = "channel_reduce"
	ChannelMoveTaskLabel   = "channel_move"

	LeaderGrowTaskLabel   = "leader_grow"
	LeaderReduceTaskLabel = "leader_reduce"
	LeaderUpdateTaskLabel = "leader_update"

	UnknownTaskLabel = "unknown"

	QueryCoordTaskType = "querycoord_task_type"
)

const (
	WALAccessModelRemote                    = "remote"
	WALAccessModelLocal                     = "local"
	WALScannerModelCatchup                  = "catchup"
	WALScannerModelTailing                  = "tailing"
	StreamingServiceClientStatusAvailable   = "available"
	StreamingServiceClientStatusUnavailable = "unavailable"
	WALStatusOK                             = "ok"
	WALStatusCancel                         = "cancel"
	WALStatusError                          = "error"

	BroadcasterTaskStateLabelName     = "state"
	ResourceKeyDomainLabelName        = "domain"
	WALAccessModelLabelName           = "access_model"
	WALScannerModelLabelName          = "scanner_model"
	TimeTickSyncTypeLabelName         = "type"
	TimeTickAckTypeLabelName          = "type"
	WALInterceptorLabelName           = "interceptor_name"
	WALTxnStateLabelName              = "state"
	WALFlusherStateLabelName          = "state"
	WALRecoveryStorageStateLabelName  = "state"
	WALStateLabelName                 = "state"
	WALChannelLabelName               = channelNameLabelName
	WALSegmentLevelLabelName          = "lv"
	WALSegmentSealPolicyNameLabelName = "policy"
	WALMessageTypeLabelName           = "message_type"
	WALChannelTermLabelName           = "term"
	WALNameLabelName                  = "wal_name"
	WALTxnTypeLabelName               = "txn_type"
	StatusLabelName                   = statusLabelName
	StreamingNodeLabelName            = "streaming_node"
	NodeIDLabelName                   = nodeIDLabelName
)

Variables

var (
	ActiveFutureTotal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: subsystemCGO,
			Name:      "active_future_total",
			Help:      "Total number of active futures.",
		}, []string{
			nodeIDLabelName,
		},
	)

	RunningCgoCallTotal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: subsystemCGO,
			Name:      "running_cgo_call_total",
			Help:      "Total number of running cgo calls.",
		}, []string{
			nodeIDLabelName,
		})

	CGODuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: subsystemCGO,
			Name:      "cgo_duration_seconds",
			Help:      "Histogram of cgo call duration in seconds.",
			Buckets:   bucketsForCGOCall,
		}, []string{
			nodeIDLabelName,
			cgoLabelName,
		},
	)

	CGOQueueDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: subsystemCGO,
			Name:      "cgo_queue_duration_seconds",
			Help:      "Duration of cgo call in queue.",
			Buckets:   bucketsForCGOCall,
		}, []string{
			nodeIDLabelName,
		},
	)
)

var (
	// DataCoordNumDataNodes records the num of data nodes managed by DataCoord.
	DataCoordNumDataNodes = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "datanode_num",
			Help:      "number of data nodes",
		}, []string{})

	DataCoordNumSegments = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "segment_num",
			Help:      "number of segments",
		}, []string{
			segmentStateLabelName,
			segmentLevelLabelName,
			segmentIsSortedLabelName,
		})

	// DataCoordCollectionNum records the num of collections managed by DataCoord.
	DataCoordNumCollections = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "collection_num",
			Help:      "number of collections",
		}, []string{})

	DataCoordSizeStoredL0Segment = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "store_level0_segment_size",
			Help:      "stored l0 segment size",
			Buckets:   buckets,
		}, []string{
			collectionIDLabelName,
		})

	DataCoordL0DeleteEntriesNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "l0_delete_entries_num",
			Help:      "Delete entries number of Level zero segment",
		}, []string{
			databaseLabelName,
			collectionIDLabelName,
		})

	// DataCoordNumStoredRows all metrics will be cleaned up after removing matched collectionID and
	// segment state labels in CleanupDataCoordNumStoredRows method.
	DataCoordNumStoredRows = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "stored_rows_num",
			Help:      "number of stored rows of healthy segment",
		}, []string{
			databaseLabelName,
			collectionIDLabelName,
			collectionName,
			segmentStateLabelName,
		})

	DataCoordBulkVectors = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "bulk_insert_vectors_count",
			Help:      "counter of vectors successfully bulk inserted",
		}, []string{
			databaseLabelName,
			collectionIDLabelName,
		})

	DataCoordConsumeDataNodeTimeTickLag = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "consume_datanode_tt_lag_ms",
			Help:      "now time minus tt per physical channel",
		}, []string{
			nodeIDLabelName,
			channelNameLabelName,
		})

	DataCoordCheckpointUnixSeconds = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "channel_checkpoint_unix_seconds",
			Help:      "channel checkpoint timestamp in unix seconds",
		}, []string{
			nodeIDLabelName,
			channelNameLabelName,
		})

	DataCoordStoredBinlogSize = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "stored_binlog_size",
			Help:      "binlog size of healthy segments",
		}, []string{
			databaseLabelName,
			collectionIDLabelName,
			segmentStateLabelName,
		})
	DataCoordSegmentBinLogFileCount = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "segment_binlog_file_count",
			Help:      "number of binlog files for each segment",
		}, []string{
			collectionIDLabelName,
		})

	DataCoordStoredIndexFilesSize = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "stored_index_files_size",
			Help:      "index files size of the segments",
		}, []string{
			databaseLabelName,
			collectionName,
			collectionIDLabelName,
		})

	DataCoordDmlChannelNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "watched_dml_chanel_num",
			Help:      "the num of dml channel watched by datanode",
		}, []string{
			nodeIDLabelName,
		})

	DataCoordCompactedSegmentSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "compacted_segment_size",
			Help:      "the segment size of compacted segment",
			Buckets:   sizeBuckets,
		}, []string{})

	DataCoordCompactionTaskNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "compaction_task_num",
			Help:      "Number of compaction tasks currently",
		}, []string{
			nodeIDLabelName,
			compactionTypeLabelName,
			statusLabelName,
		})

	DataCoordCompactionLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "compaction_latency",
			Help:      "latency of compaction operation",
			Buckets:   longTaskBuckets,
		}, []string{
			isVectorFieldLabelName,
			channelNameLabelName,
			compactionTypeLabelName,
			stageLabelName,
		})

	ImportJobLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "import_job_latency",
			Help:      "latency of import job",
			Buckets:   longTaskBuckets,
		}, []string{
			importStageLabelName,
		})

	ImportTaskLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "import_task_latency",
			Help:      "latency of import task",
			Buckets:   longTaskBuckets,
		}, []string{
			importStageLabelName,
		})

	FlushedSegmentFileNum = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Name:      "flushed_segment_file_num",
			Help:      "the num of files for flushed segment",
			Buckets:   buckets,
		}, []string{segmentFileTypeLabelName})

	// GarbageCollectorFileScanDuration metrics for gc scan storage files.
	GarbageCollectorFileScanDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "gc_file_scan_duration",
			Help:      "duration of scan file in storage while garbage collecting (in milliseconds)",
			Buckets:   longTaskBuckets,
		}, []string{nodeIDLabelName, segmentFileTypeLabelName})

	GarbageCollectorRunCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "gc_run_count",
			Help:      "garbage collection running count",
		}, []string{nodeIDLabelName})

	// IndexRequestCounter records the number of the index requests.
	IndexRequestCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "index_req_count",
			Help:      "number of building index requests ",
		}, []string{statusLabelName})

	// IndexTaskNum records the number of index tasks of each type.
	// Deprecated: please ues TaskNum after v2.5.5.
	IndexTaskNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "index_task_count",
			Help:      "number of index tasks of each type",
		}, []string{collectionIDLabelName, indexTaskStatusLabelName})

	// IndexNodeNum records the number of IndexNodes managed by IndexCoord.
	IndexNodeNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "index_node_num",
			Help:      "number of IndexNodes managed by IndexCoord",
		}, []string{})

	ImportJobs = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "import_jobs",
			Help:      "the import jobs grouping by state",
		}, []string{"import_state"})

	ImportTasks = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "import_tasks",
			Help:      "the import tasks grouping by type and state",
		}, []string{"task_type", "import_state"})

	DataCoordTaskExecuteLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "task_execute_max_latency",
			Help:      "latency of task execute operation",
			Buckets:   longTaskBuckets,
		}, []string{
			TaskTypeLabel,
			statusLabelName,
		})

	// IndexStatsTaskNum records the number of tasks of each type.
	IndexStatsTaskNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "task_count",
			Help:      "number of index tasks of each type",
		}, []string{TaskTypeLabel, TaskStateLabel})

	// TaskVersion records the version of task(retry times of task).
	TaskVersion = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "task_version",
			Help:      "version of task",
			Buckets:   buckets,
		}, []string{
			TaskTypeLabel,
		})

	TaskNumInGlobalScheduler = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataCoordRole,
			Name:      "task_num_in_scheduler",
			Help:      "number of tasks in global scheduler",
		}, []string{TaskTypeLabel, TaskStateLabel})
)

var (
	DataNodeNumFlowGraphs = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "flowgraph_num",
			Help:      "number of flowgraphs",
		}, []string{
			nodeIDLabelName,
		})

	DataNodeConsumeMsgRowsCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "msg_rows_count",
			Help:      "count of rows consumed from msgStream",
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
		})

	DataNodeFlushedSize = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "flushed_data_size",
			Help:      "byte size of data flushed to storage",
		}, []string{
			nodeIDLabelName,
			dataSourceLabelName,
			segmentLevelLabelName,
		})

	DataNodeWriteDataCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "write_data_count",
			Help:      "byte size of datanode write to object storage, including flushed size",
		}, []string{
			nodeIDLabelName,
			dataSourceLabelName,
			dataTypeLabelName,
			collectionIDLabelName,
		})

	DataNodeFlushedRows = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "flushed_data_rows",
			Help:      "num of rows flushed to storage",
		}, []string{
			nodeIDLabelName,
			dataSourceLabelName,
		})

	DataNodeNumProducers = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "producer_num",
			Help:      "number of producers",
		}, []string{
			nodeIDLabelName,
		})

	DataNodeConsumeTimeTickLag = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "consume_tt_lag_ms",
			Help:      "now time minus tt per physical channel",
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
			collectionIDLabelName,
		})

	DataNodeConsumeMsgCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "consume_msg_count",
			Help:      "count of consumed msg",
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
			collectionIDLabelName,
		})

	DataNodeSave2StorageLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "save_latency",
			Help:      "latency of saving flush data to storage",
			Buckets:   []float64{0, 10, 100, 200, 400, 1000, 10000},
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
		})

	DataNodeFlushBufferCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "flush_buffer_op_count",
			Help:      "count of flush buffer operations",
		}, []string{
			nodeIDLabelName,
			statusLabelName,
			segmentLevelLabelName,
		})

	DataNodeAutoFlushBufferCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "autoflush_buffer_op_count",
			Help:      "count of auto flush buffer operations",
		}, []string{
			nodeIDLabelName,
			statusLabelName,
			segmentLevelLabelName,
		})

	DataNodeCompactionLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "compaction_latency",
			Help:      "latency of compaction operation",
			Buckets:   longTaskBuckets,
		}, []string{
			nodeIDLabelName,
			compactionTypeLabelName,
		})

	DataNodeCompactionLatencyInQueue = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "compaction_latency_in_queue",
			Help:      "latency of compaction operation in queue",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
		})

	// DataNodeFlushReqCounter counts the num of calls of FlushSegments
	DataNodeFlushReqCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "flush_req_count",
			Help:      "count of flush request",
		}, []string{
			nodeIDLabelName,
			statusLabelName,
		})

	// DataNodeConsumeBytesCount counts the bytes DataNode consumed from message storage.
	DataNodeConsumeBytesCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "consume_bytes_count",
			Help:      "",
		}, []string{nodeIDLabelName, msgTypeLabelName})

	DataNodeForwardDeleteMsgTimeTaken = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "forward_delete_msg_time_taken_ms",
			Help:      "forward delete message time taken",
			Buckets:   buckets,
		}, []string{nodeIDLabelName})

	DataNodeFlowGraphBufferDataSize = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "fg_buffer_size",
			Help:      "the buffered data size of flow graph",
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
		})

	DataNodeMsgDispatcherTtLag = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "msg_dispatcher_tt_lag_ms",
			Help:      "time.Now() sub dispatcher's current consume time",
		}, []string{
			nodeIDLabelName,
			channelNameLabelName,
		})

	DataNodeCompactionDeleteCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "compaction_delete_count",
			Help:      "Number of delete entries in compaction",
		}, []string{collectionIDLabelName})

	DataNodeCompactionMissingDeleteCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "compaction_missing_delete_count",
			Help:      "Number of missing deletes in compaction",
		}, []string{collectionIDLabelName})

	DataNodeBuildIndexTaskCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "index_task_count",
			Help:      "number of tasks that index node received",
		}, []string{nodeIDLabelName, statusLabelName})

	DataNodeLoadFieldLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "load_field_latency",
			Help:      "latency of loading the field data",
			Buckets:   indexBucket,
		}, []string{nodeIDLabelName})

	DataNodeDecodeFieldLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "decode_field_latency",
			Help:      "latency of decode field data",
			Buckets:   indexBucket,
		}, []string{nodeIDLabelName})

	DataNodeKnowhereBuildIndexLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "knowhere_build_index_latency",
			Help:      "latency of building the index by knowhere",
			Buckets:   indexBucket,
		}, []string{nodeIDLabelName})

	DataNodeEncodeIndexFileLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "encode_index_latency",
			Help:      "latency of encoding the index file",
			Buckets:   indexBucket,
		}, []string{nodeIDLabelName})

	DataNodeSaveIndexFileLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "save_index_latency",
			Help:      "latency of saving the index file",
			Buckets:   indexBucket,
		}, []string{nodeIDLabelName})

	DataNodeIndexTaskLatencyInQueue = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "index_task_latency_in_queue",
			Help:      "latency of index task in queue",
			Buckets:   buckets,
		}, []string{nodeIDLabelName})

	DataNodeBuildIndexLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "build_index_latency",
			Help:      "latency of build index for segment",
			Buckets:   indexBucket,
		}, []string{nodeIDLabelName})

	DataNodeBuildJSONStatsLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.IndexNodeRole,
			Name:      "task_build_json_stats_latency",
			Help:      "latency of building the index by knowhere",
			Buckets:   indexBucket,
		}, []string{nodeIDLabelName})

	DataNodeSlot = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.DataNodeRole,
			Name:      "slot",
			Help:      "number of available and used slot",
		}, []string{nodeIDLabelName, "type"})
)

var (
	BuildInfo = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Name:      "build_info",
			Help:      "Build information of milvus",
		},
		[]string{
			"version",
			"built",
			"git_commit",
		},
	)

	RuntimeInfo = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Name:      "runtime_info",
			Help:      "Runtime information of milvus",
		},
		[]string{
			"mq",
			"meta",
		},
	)

	ThreadNum = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Name:      "thread_num",
			Help:      "the actual thread number of milvus process",
		},
	)
)

var (
	MetaKvSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: "meta",
			Name:      "kv_size",
			Help:      "kv size stats",
			Buckets:   buckets,
		}, []string{metaOpType})

	MetaRequestLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: "meta",
			Name:      "request_latency",
			Help:      "request latency on the client side ",
			Buckets:   buckets,
		}, []string{metaOpType})

	MetaOpCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: "meta",
			Name:      "op_count",
			Help:      "count of meta operation",
		}, []string{metaOpType, statusLabelName})
)

var (
	NumNodes = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Name:      "num_node",
			Help:      "number of nodes and coordinates",
		}, []string{nodeIDLabelName, roleNameLabelName})

	LockCosts = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Name:      "lock_time_cost",
			Help:      "time cost for various kinds of locks",
		}, []string{
			lockName,
			lockSource,
			lockType,
			lockOp,
		})
)

var (
	NumConsumers = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: "msg_queue",
			Name:      "consumer_num",
			Help:      "number of consumers",
		}, []string{
			roleNameLabelName,
			nodeIDLabelName,
		})

	MsgStreamRequestLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: "msgstream",
			Name:      "request_latency",
			Help:      "request latency on the client side ",
			Buckets:   buckets,
		}, []string{msgStreamOpType})

	MsgStreamOpCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: "msgstream",
			Name:      "op_count",
			Help:      "count of stream message operation",
		}, []string{msgStreamOpType, statusLabelName})
)

var (
	PersistentDataKvSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: "storage",
			Name:      "kv_size",
			Help:      "kv size stats",
			Buckets:   buckets,
		}, []string{persistentDataOpType})

	PersistentDataRequestLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: "storage",
			Name:      "request_latency",
			Help:      "request latency on the client side ",
			Buckets:   buckets,
		}, []string{persistentDataOpType})

	PersistentDataOpCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: "storage",
			Name:      "op_count",
			Help:      "count of persistent data operation",
		}, []string{persistentDataOpType, statusLabelName})
)

var (
	ProxyReceivedNQ = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "received_nq",
			Help:      "counter of nq of received search and query requests",
		}, []string{nodeIDLabelName, queryTypeLabelName, databaseLabelName, collectionName})

	// ProxySearchVectors record the number of vectors search successfully.
	ProxySearchVectors = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "search_vectors_count",
			Help:      "counter of vectors successfully searched",
		}, []string{nodeIDLabelName, databaseLabelName, collectionName})

	// ProxyInsertVectors record the number of vectors insert successfully.
	ProxyInsertVectors = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "insert_vectors_count",
			Help:      "counter of vectors successfully inserted",
		}, []string{nodeIDLabelName, databaseLabelName, collectionName})

	// ProxyUpsertVectors record the number of vectors upsert successfully.
	ProxyUpsertVectors = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "upsert_vectors_count",
			Help:      "counter of vectors successfully upserted",
		}, []string{nodeIDLabelName, databaseLabelName, collectionName})

	ProxyDeleteVectors = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "delete_vectors_count",
			Help:      "counter of vectors successfully deleted",
		}, []string{nodeIDLabelName, databaseLabelName, collectionName})

	// ProxySQLatency record the latency of search successfully.
	ProxySQLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "sq_latency",
			Help:      "latency of search or query successfully",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, queryTypeLabelName, databaseLabelName, collectionName})

	// ProxyCollectionSQLatency record the latency of search successfully, per collection
	// Deprecated, ProxySQLatency instead of it
	ProxyCollectionSQLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "collection_sq_latency",
			Help:      "latency of search or query successfully, per collection",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, queryTypeLabelName, databaseLabelName, collectionName})

	// ProxyMutationLatency record the latency that mutate successfully.
	ProxyMutationLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "mutation_latency",
			Help:      "latency of insert or delete successfully",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, msgTypeLabelName, databaseLabelName, collectionName})

	// ProxyCollectionMutationLatency record the latency that mutate successfully, per collection
	// Deprecated, ProxyMutationLatency instead of it
	ProxyCollectionMutationLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "collection_mutation_latency",
			Help:      "latency of insert or delete successfully, per collection",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, msgTypeLabelName, databaseLabelName, collectionName})

	// ProxyWaitForSearchResultLatency record the time that the proxy waits for the search result.
	ProxyWaitForSearchResultLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "sq_wait_result_latency",
			Help:      "latency that proxy waits for the result",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, queryTypeLabelName})
	// ProxyReduceResultLatency record the time that the proxy reduces search result.
	ProxyReduceResultLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "sq_reduce_result_latency",
			Help:      "latency that proxy reduces search result",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, queryTypeLabelName})

	// ProxyDecodeResultLatency record the time that the proxy decodes the search result.
	ProxyDecodeResultLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "sq_decode_result_latency",
			Help:      "latency that proxy decodes the search result",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, queryTypeLabelName})

	// ProxyMsgStreamObjectsForPChan record the number of MsgStream objects per PChannel on each collection_id on Proxy.
	ProxyMsgStreamObjectsForPChan = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "msgstream_obj_num",
			Help:      "number of MsgStream objects per physical channel",
		}, []string{nodeIDLabelName, channelNameLabelName})

	// ProxySendMutationReqLatency record the latency that Proxy send insert request to MsgStream.
	ProxySendMutationReqLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "mutation_send_latency",
			Help:      "latency that proxy send insert request to MsgStream",
			Buckets:   longTaskBuckets,
		}, []string{nodeIDLabelName, msgTypeLabelName})

	// ProxyAssignSegmentIDLatency record the latency that Proxy get segmentID from dataCoord.
	ProxyAssignSegmentIDLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "assign_segmentID_latency",
			Help:      "latency that proxy get segmentID from dataCoord",
			Buckets:   buckets,
		}, []string{nodeIDLabelName})

	// ProxySyncSegmentRequestLength the length of SegmentIDRequests when assigning segments for insert.
	ProxySyncSegmentRequestLength = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "sync_segment_request_length",
			Help:      "the length of SegmentIDRequests when assigning segments for insert",
			Buckets:   buckets,
		}, []string{nodeIDLabelName})

	// ProxyCacheStatsCounter record the number of Proxy cache hits or miss.
	ProxyCacheStatsCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "cache_hit_count",
			Help:      "count of cache hits/miss",
		}, []string{nodeIDLabelName, cacheNameLabelName, cacheStateLabelName})

	// ProxyUpdateCacheLatency record the time that proxy update cache when cache miss.
	ProxyUpdateCacheLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "cache_update_latency",
			Help:      "latency that proxy update cache when cache miss",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, cacheNameLabelName})

	// ProxySyncTimeTickLag record Proxy synchronization timestamp statistics, differentiated by Channel.
	ProxySyncTimeTickLag = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "tt_lag_ms",
			Help:      "now time minus tt per physical channel",
		}, []string{nodeIDLabelName, channelNameLabelName})

	// ProxyApplyPrimaryKeyLatency record the latency that apply primary key.
	ProxyApplyPrimaryKeyLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "apply_pk_latency",
			Help:      "latency that apply primary key",
			Buckets:   buckets,
		}, []string{nodeIDLabelName})

	// ProxyApplyTimestampLatency record the latency that proxy apply timestamp.
	ProxyApplyTimestampLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "apply_timestamp_latency",
			Help:      "latency that proxy apply timestamp",
			Buckets:   buckets,
		}, []string{nodeIDLabelName})

	// ProxyFunctionCall records the number of times the function of the DDL operation was executed, like `CreateCollection`.
	ProxyFunctionCall = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "req_count",
			Help:      "count of operation executed",
		}, []string{nodeIDLabelName, functionLabelName, statusLabelName, databaseLabelName, collectionName})

	// ProxyReqLatency records the latency for each grpc request.
	ProxyGRPCLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "grpc_latency",
			Help:      "latency of each grpc request",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, functionLabelName, statusLabelName})

	// ProxyReqLatency records the latency that for all requests, like "CreateCollection".
	ProxyReqLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "req_latency",
			Help:      "latency of each request",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, functionLabelName})

	// ProxyReceiveBytes record the received bytes of messages in Proxy
	ProxyReceiveBytes = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "receive_bytes_count",
			Help:      "count of bytes received  from sdk",
		}, []string{nodeIDLabelName, msgTypeLabelName, databaseLabelName, collectionName})

	// ProxyReadReqSendBytes record the bytes sent back to client by Proxy
	ProxyReadReqSendBytes = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "send_bytes_count",
			Help:      "count of bytes sent back to sdk",
		}, []string{nodeIDLabelName})

	// RestfulFunctionCall records the number of times the restful apis was called.
	RestfulFunctionCall = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "restful_api_req_count",
			Help:      "count of operation executed",
		}, []string{nodeIDLabelName, pathLabelName})

	// RestfulReqLatency records the latency that for all requests.
	RestfulReqLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "restful_api_req_latency",
			Help:      "latency of each request",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, pathLabelName})

	// RestfulReceiveBytes record the received bytes of messages in Proxy
	RestfulReceiveBytes = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "restful_api_receive_bytes_count",
			Help:      "count of bytes received  from sdk",
		}, []string{nodeIDLabelName, pathLabelName})

	// RestfulSendBytes record the bytes sent back to client by Proxy
	RestfulSendBytes = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "restful_api_send_bytes_count",
			Help:      "count of bytes sent back to sdk",
		}, []string{nodeIDLabelName, pathLabelName})

	// ProxyReportValue records value about the request
	ProxyReportValue = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "report_value",
			Help:      "report value about the request",
		}, []string{nodeIDLabelName, msgTypeLabelName, databaseLabelName, usernameLabelName})

	// ProxyLimiterRate records rates of rateLimiter in Proxy.
	ProxyLimiterRate = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "limiter_rate",
			Help:      "",
		}, []string{nodeIDLabelName, collectionIDLabelName, msgTypeLabelName})

	ProxyHookFunc = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "hook_func_count",
			Help:      "the hook function count",
		}, []string{functionLabelName, fullMethodLabelName})

	UserRPCCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "user_rpc_count",
			Help:      "the rpc count of a user",
		}, []string{usernameLabelName})

	// ProxyWorkLoadScore record the score that measured query node's workload.
	ProxyWorkLoadScore = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "workload_score",
			Help:      "score that measured query node's workload",
		}, []string{
			nodeIDLabelName,
		})

	ProxyExecutingTotalNq = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "executing_total_nq",
			Help:      "total nq of executing search/query",
		}, []string{
			nodeIDLabelName,
		})

	// ProxyRateLimitReqCount integrates a counter monitoring metric for the rate-limit rpc requests.
	ProxyRateLimitReqCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "rate_limit_req_count",
			Help:      "count of operation executed",
		}, []string{nodeIDLabelName, msgTypeLabelName, statusLabelName})

	ProxySlowQueryCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "slow_query_count",
			Help:      "count of slow query executed",
		}, []string{nodeIDLabelName, msgTypeLabelName})

	// ProxyReqInQueueLatency records the latency that requests wait in the queue, like "CreateCollection".
	ProxyReqInQueueLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "req_in_queue_latency",
			Help:      "latency which request waits in the queue",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, functionLabelName})

	MaxInsertRate = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "max_insert_rate",
			Help:      "max insert rate",
		}, []string{"node_id", "scope"})

	// ProxyRetrySearchCount records the retry search count when result count does not meet limit and topk reduce is on
	ProxyRetrySearchCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "retry_search_cnt",
			Help:      "counter of retry search",
		}, []string{nodeIDLabelName, queryTypeLabelName, databaseLabelName, collectionName})

	// ProxyRetrySearchResultInsufficientCount records the retry search without reducing topk that still not meet result limit
	// there are more likely some non-index-related reasons like we do not have enough entities for very big k, duplicate pks, etc
	ProxyRetrySearchResultInsufficientCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "retry_search_result_insufficient_cnt",
			Help:      "counter of retry search which does not have enough results",
		}, []string{nodeIDLabelName, queryTypeLabelName, databaseLabelName, collectionName})

	// ProxyRecallSearchCount records the counter that users issue recall evaluation requests, which are cpu-intensive
	ProxyRecallSearchCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "recall_search_cnt",
			Help:      "counter of recall search",
		}, []string{nodeIDLabelName, queryTypeLabelName, databaseLabelName, collectionName})

	// ProxySearchSparseNumNonZeros records the estimated number of non-zeros in each sparse search task
	ProxySearchSparseNumNonZeros = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "search_sparse_num_non_zeros",
			Help:      "the number of non-zeros in each sparse search task",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, collectionName, queryTypeLabelName, fieldIDLabelName})

	// ProxyQueueTaskNum records task number of queue in Proxy.
	ProxyQueueTaskNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "queue_task_num",
			Help:      "",
		}, []string{nodeIDLabelName, queueTypeLabelName, TaskStateLabel})

	ProxyParseExpressionLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "parse_expr_latency",
			Help:      "the latency of parse expression",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, functionLabelName, statusLabelName})
	// ProxyFunctionlatency records the latency of function
	ProxyFunctionlatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.ProxyRole,
			Name:      "function_udf_call_latency",
			Help:      "latency of function call",
			Buckets:   buckets,
		}, []string{nodeIDLabelName, collectionName, functionTypeName, functionProvider, functionName})
)

var (
	QueryCoordNumCollections = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "collection_num",
			Help:      "number of collections",
		}, []string{})

	QueryCoordNumPartitions = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "partition_num",
			Help:      "number of partitions",
		}, []string{})

	QueryCoordLoadCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "load_req_count",
			Help:      "count of load request",
		}, []string{
			statusLabelName,
		})

	QueryCoordReleaseCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "release_req_count",
			Help:      "count of release request",
		}, []string{
			statusLabelName,
		})

	QueryCoordLoadLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "load_latency",
			Help:      "latency of load the entire collection",
			Buckets:   []float64{0, 500, 1000, 2000, 5000, 10000, 20000, 50000, 60000, 300000, 600000, 1800000},
		}, []string{})

	QueryCoordReleaseLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "release_latency",
			Help:      "latency of release request",
			Buckets:   []float64{0, 5, 10, 20, 40, 100, 200, 400, 1000, 10000},
		}, []string{})

	QueryCoordTaskNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "task_num",
			Help:      "the number of tasks in QueryCoord's scheduler",
		}, []string{QueryCoordTaskType})

	QueryCoordNumQueryNodes = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "querynode_num",
			Help:      "number of QueryNodes managered by QueryCoord",
		}, []string{})

	QueryCoordCurrentTargetCheckpointUnixSeconds = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "current_target_checkpoint_unix_seconds",
			Help:      "current target checkpoint timestamp in unix seconds",
		}, []string{
			nodeIDLabelName,
			channelNameLabelName,
		})

	QueryCoordTaskLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "task_latency",
			Help:      "latency of all kind of task in query coord scheduler scheduler",
			Buckets:   longTaskBuckets,
		}, []string{collectionIDLabelName, TaskTypeLabel, channelNameLabelName})

	QueryCoordResourceGroupInfo = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "resource_group_info",
			Help:      "all resource group detail info in query coord",
		}, []string{ResourceGroupLabelName, NodeIDLabelName})

	QueryCoordResourceGroupReplicaTotal = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "resource_group_replica_total",
			Help:      "total replica number of resource group",
		}, []string{ResourceGroupLabelName})

	QueryCoordReplicaRONodeTotal = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "replica_ro_node_total",
			Help:      "total read only node number of replica",
		})

	QueryCoordLastHeartbeatTimeStamp = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryCoordRole,
			Name:      "last_heartbeat_timestamp",
			Help:      "heartbeat timestamp of query node",
		}, []string{nodeIDLabelName})
)

var (
	QueryNodeNumCollections = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "collection_num",
			Help:      "number of collections loaded",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeConsumeTimeTickLag = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "consume_tt_lag_ms",
			Help:      "now time minus tt per physical channel",
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
			collectionIDLabelName,
		})

	QueryNodeProcessCost = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "process_insert_or_delete_latency",
			Help:      "process insert or delete cost in ms",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
		})

	QueryNodeApplyBFCost = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "apply_bf_latency",
			Help:      "apply bf cost in ms",
			Buckets:   buckets,
		}, []string{
			functionLabelName,
			nodeIDLabelName,
		})

	QueryNodeForwardDeleteCost = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "forward_delete_latency",
			Help:      "forward delete cost in ms",
			Buckets:   buckets,
		}, []string{
			functionLabelName,
			nodeIDLabelName,
		})

	QueryNodeWaitProcessingMsgCount = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "wait_processing_msg_count",
			Help:      "count of wait processing msg",
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
		})

	QueryNodeConsumerMsgCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "consume_msg_count",
			Help:      "count of consumed msg",
		}, []string{
			nodeIDLabelName,
			msgTypeLabelName,
			collectionIDLabelName,
		})

	QueryNodeNumPartitions = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "partition_num",
			Help:      "number of partitions loaded",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeNumSegments = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_num",
			Help:      "number of segments loaded, clustered by its collection, partition, state and # of indexed fields",
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			segmentStateLabelName,
			segmentLevelLabelName,
		})

	QueryNodeNumDmlChannels = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "dml_vchannel_num",
			Help:      "number of dmlChannels watched",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeNumDeltaChannels = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "delta_vchannel_num",
			Help:      "number of deltaChannels watched",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeSQCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_req_count",
			Help:      "count of search / query request",
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			statusLabelName,
			requestScope,
			collectionIDLabelName,
		})

	QueryNodeSQReqLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_req_latency",
			Help:      "latency of Search or query requests",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			requestScope,
		})

	QueryNodeSQLatencyWaitTSafe = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_wait_tsafe_latency",
			Help:      "latency of search or query to wait for tsafe",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
		})

	QueryNodeSQLatencyInQueue = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_queue_latency",
			Help:      "latency of search or query in queue",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
		})

	QueryNodeSQPerUserLatencyInQueue = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_queue_user_latency",
			Help:      "latency per user of search or query in queue",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			usernameLabelName,
		},
	)

	QueryNodeSQSegmentLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_segment_latency",
			Help:      "latency of search or query per segment",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			segmentStateLabelName,
		})

	QueryNodeSQSegmentLatencyInCore = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_core_latency",
			Help:      "latency of search or query latency in segcore",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
		})

	QueryNodeReduceLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "sq_reduce_latency",
			Help:      "latency of reduce search or query result",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			reduceLevelName,
			reduceType,
		})

	QueryNodeLoadSegmentLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "load_segment_latency",
			Help:      "latency of load per segment",
			Buckets:   longTaskBuckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeReadTaskUnsolveLen = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "read_task_unsolved_len",
			Help:      "number of unsolved read tasks in unsolvedQueue",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeReadTaskReadyLen = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "read_task_ready_len",
			Help:      "number of ready read tasks in readyQueue",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeReadTaskConcurrency = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "read_task_concurrency",
			Help:      "number of concurrent executing read tasks in QueryNode",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeEstimateCPUUsage = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "estimate_cpu_usage",
			Help:      "estimated cpu usage by the scheduler in QueryNode",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeSearchGroupNQ = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "search_group_nq",
			Help:      "the number of queries of each grouped search task",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeSearchNQ = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "search_nq",
			Help:      "the number of queries of each search task",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeSearchGroupTopK = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "search_group_topk",
			Help:      "the topK of each grouped search task",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeSearchTopK = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "search_topk",
			Help:      "the top of each search task",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeSearchFTSNumTokens = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "search_fts_num_tokens",
			Help:      "number of tokens in each Full Text Search search task",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			fieldIDLabelName,
		})

	QueryNodeSearchGroupSize = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "search_group_size",
			Help:      "the number of tasks of each grouped search task",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeSearchHitSegmentNum = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "search_hit_segment_num",
			Help:      "the number of segments actually involved in search task",
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			queryTypeLabelName,
		})

	QueryNodeSegmentPruneRatio = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_prune_ratio",
			Help:      "ratio of segments pruned by segment_pruner",
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			segmentPruneLabelName,
		})

	QueryNodeSegmentPruneBias = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_prune_bias",
			Help:      "bias of workload when enabling segment prune",
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			segmentPruneLabelName,
		})

	QueryNodeSegmentPruneLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_prune_latency",
			Help:      "latency of segment prune",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			segmentPruneLabelName,
		})

	QueryNodeEvictedReadReqCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "read_evicted_count",
			Help:      "count of evicted search / query request",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeNumFlowGraphs = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "flowgraph_num",
			Help:      "number of flowgraphs",
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeNumEntities = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "entity_num",
			Help:      "number of entities which can be searched/queried, clustered by collection, partition and state",
		}, []string{
			databaseLabelName,
			collectionName,
			nodeIDLabelName,
			collectionIDLabelName,
			segmentStateLabelName,
		})

	QueryNodeEntitiesSize = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "entity_size",
			Help:      "entities' memory size, clustered by collection and state",
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			segmentStateLabelName,
		})

	QueryNodeLevelZeroSize = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "level_zero_size",
			Help:      "level zero segments' delete records memory size, clustered by collection and state",
		}, []string{
			nodeIDLabelName,
			collectionIDLabelName,
			channelNameLabelName,
		})

	// QueryNodeConsumeCounter counts the bytes QueryNode consumed from message storage.
	QueryNodeConsumeCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "consume_bytes_counter",
			Help:      "",
		}, []string{nodeIDLabelName, msgTypeLabelName})

	// QueryNodeExecuteCounter counts the bytes of requests in QueryNode.
	QueryNodeExecuteCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "execute_bytes_counter",
			Help:      "",
		}, []string{nodeIDLabelName, msgTypeLabelName})

	QueryNodeMsgDispatcherTtLag = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "msg_dispatcher_tt_lag_ms",
			Help:      "time.Now() sub dispatcher's current consume time",
		}, []string{
			nodeIDLabelName,
			channelNameLabelName,
		})

	QueryNodeSegmentSearchLatencyPerVector = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_latency_per_vector",
			Help:      "one vector's search latency per segment",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			segmentStateLabelName,
		})

	QueryNodeWatchDmlChannelLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "watch_dml_channel_latency",
			Help:      "latency of watch dml channel",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeDiskUsedSize = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "disk_used_size",
			Help:      "disk used size(MB)",
		}, []string{
			nodeIDLabelName,
		})

	StoppingBalanceNodeNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "stopping_balance_node_num",
			Help:      "the number of node which executing stopping balance",
		}, []string{})

	StoppingBalanceChannelNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "stopping_balance_channel_num",
			Help:      "the number of channel which executing stopping balance",
		}, []string{nodeIDLabelName})

	StoppingBalanceSegmentNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "stopping_balance_segment_num",
			Help:      "the number of segment which executing stopping balance",
		}, []string{nodeIDLabelName})

	QueryNodeLoadSegmentConcurrency = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "load_segment_concurrency",
			Help:      "number of concurrent loading segments in QueryNode",
		}, []string{
			nodeIDLabelName,
			loadTypeName,
		})

	QueryNodeLoadIndexLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "load_index_latency",
			Help:      "latency of load per segment's index, in milliseconds",
			Buckets:   longTaskBuckets,
		}, []string{
			nodeIDLabelName,
		})

	// QueryNodeSegmentAccessTotal records the total number of search or query segments accessed.
	QueryNodeSegmentAccessTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_access_total",
			Help:      "number of segments accessed",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
			queryTypeLabelName,
		},
	)

	// QueryNodeSegmentAccessDuration records the total time cost of accessing segments including cache loads.
	QueryNodeSegmentAccessDuration = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_access_duration",
			Help:      "total time cost of accessing segments",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
			queryTypeLabelName,
		},
	)

	// QueryNodeSegmentAccessGlobalDuration records the global time cost of accessing segments.
	QueryNodeSegmentAccessGlobalDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_access_global_duration",
			Help:      "global time cost of accessing segments",
			Buckets:   longTaskBuckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
		},
	)

	// QueryNodeSegmentAccessWaitCacheTotal records the number of search or query segments that have to wait for loading access.
	QueryNodeSegmentAccessWaitCacheTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_access_wait_cache_total",
			Help:      "number of segments waiting for loading access",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
			queryTypeLabelName,
		})

	// QueryNodeSegmentAccessWaitCacheDuration records the total time cost of waiting for loading access.
	QueryNodeSegmentAccessWaitCacheDuration = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_access_wait_cache_duration",
			Help:      "total time cost of waiting for loading access",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
			queryTypeLabelName,
		})

	// QueryNodeSegmentAccessWaitCacheGlobalDuration records the global time cost of waiting for loading access.
	QueryNodeSegmentAccessWaitCacheGlobalDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "segment_access_wait_cache_global_duration",
			Help:      "global time cost of waiting for loading access",
			Buckets:   longTaskBuckets,
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
		})

	// QueryNodeDiskCacheLoadTotal records the number of real segments loaded from disk cache.
	QueryNodeDiskCacheLoadTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Help:      "number of segments loaded from disk cache",
			Name:      "disk_cache_load_total",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
		})

	// QueryNodeDiskCacheLoadBytes records the number of bytes loaded from disk cache.
	QueryNodeDiskCacheLoadBytes = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Help:      "number of bytes loaded from disk cache",
			Name:      "disk_cache_load_bytes",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
		})

	// QueryNodeDiskCacheLoadDuration records the total time cost of loading segments from disk cache.
	// With db and resource group labels.
	QueryNodeDiskCacheLoadDuration = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Help:      "total time cost of loading segments from disk cache",
			Name:      "disk_cache_load_duration",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
		})

	// QueryNodeDiskCacheLoadGlobalDuration records the global time cost of loading segments from disk cache.
	QueryNodeDiskCacheLoadGlobalDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "disk_cache_load_global_duration",
			Help:      "global duration of loading segments from disk cache",
			Buckets:   longTaskBuckets,
		}, []string{
			nodeIDLabelName,
		})

	// QueryNodeDiskCacheEvictTotal records the number of real segments evicted from disk cache.
	QueryNodeDiskCacheEvictTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "disk_cache_evict_total",
			Help:      "number of segments evicted from disk cache",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
		})

	// QueryNodeDiskCacheEvictBytes records the number of bytes evicted from disk cache.
	QueryNodeDiskCacheEvictBytes = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "disk_cache_evict_bytes",
			Help:      "number of bytes evicted from disk cache",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
		})

	// QueryNodeDiskCacheEvictDuration records the total time cost of evicting segments from disk cache.
	QueryNodeDiskCacheEvictDuration = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "disk_cache_evict_duration",
			Help:      "total time cost of evicting segments from disk cache",
		}, []string{
			nodeIDLabelName,
			databaseLabelName,
			ResourceGroupLabelName,
		})

	// QueryNodeDiskCacheEvictGlobalDuration records the global time cost of evicting segments from disk cache.
	QueryNodeDiskCacheEvictGlobalDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "disk_cache_evict_global_duration",
			Help:      "global duration of evicting segments from disk cache",
			Buckets:   longTaskBuckets,
		}, []string{
			nodeIDLabelName,
		})

	QueryNodeDeleteBufferSize = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "delete_buffer_size",
			Help:      "delegator delete buffer size (in bytes)",
		}, []string{
			nodeIDLabelName,
			channelNameLabelName,
		},
	)

	QueryNodeDeleteBufferRowNum = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "delete_buffer_row_num",
			Help:      "delegator delete buffer row num",
		}, []string{
			nodeIDLabelName,
			channelNameLabelName,
		},
	)

	QueryNodeCGOCallLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "cgo_latency",
			Help:      "latency of each cgo call",
			Buckets:   buckets,
		}, []string{
			nodeIDLabelName,
			cgoNameLabelName,
			cgoTypeLabelName,
		})

	QueryNodePartialResultCount = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.QueryNodeRole,
			Name:      "partial_result_count",
			Help:      "count of partial result",
		}, []string{
			nodeIDLabelName,
			queryTypeLabelName,
			collectionIDLabelName,
		})
)

var (
	// RootCoordProxyCounter counts the num of registered proxy nodes
	RootCoordProxyCounter = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "proxy_num",
			Help:      "number of proxy nodes managered by rootcoord",
		}, []string{})

	// RootCoordInsertChannelTimeTick counts the time tick num of insert channel in 24H
	RootCoordInsertChannelTimeTick = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "produce_tt_lag_ms",
			Help:      "now time minus tt per physical channel",
		}, []string{channelNameLabelName})

	RootCoordDDLReqCounter = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "ddl_req_count",
			Help:      "count of DDL operations",
		}, []string{functionLabelName, statusLabelName})

	// RootCoordDDLReqLatency records the latency for read type of DDL operations.
	RootCoordDDLReqLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "ddl_req_latency",
			Help:      "latency of each DDL operations",
			Buckets:   buckets,
		}, []string{functionLabelName})

	// RootCoordSyncTimeTickLatency records the latency of sync time tick.
	RootCoordSyncTimeTickLatency = prometheus.NewHistogram(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "sync_timetick_latency",
			Help:      "latency of synchronizing timetick message",
			Buckets:   buckets,
		})

	// RootCoordIDAllocCounter records the number of global ID allocations.
	RootCoordIDAllocCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "id_alloc_count",
			Help:      "count of ID allocated",
		})

	// RootCoordTimestamp records the number of timestamp allocations in RootCoord.
	RootCoordTimestamp = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "timestamp",
			Help:      "lateste timestamp allocated in memory",
		})

	// RootCoordTimestampSaved records the number of timestamp allocations in ETCD.
	RootCoordTimestampSaved = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "timestamp_saved",
			Help:      "timestamp saved in meta storage",
		})

	// RootCoordNumOfDatabases counts the number of database.
	RootCoordNumOfDatabases = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "database_num",
			Help:      "number of database",
		})

	// RootCoordNumOfCollections counts the number of collections.
	RootCoordNumOfCollections = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "collection_num",
			Help:      "number of collections",
		}, []string{databaseLabelName})

	// RootCoordNumOfPartitions counts the number of partitions per collection.
	RootCoordNumOfPartitions = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "partition_num",
			Help:      "number of partitions",
		}, []string{})

	// RootCoordNumOfDMLChannel counts the number of DML channels.
	RootCoordNumOfDMLChannel = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "dml_channel_num",
			Help:      "number of DML channels",
		})

	// RootCoordNumOfMsgStream counts the number of message streams.
	RootCoordNumOfMsgStream = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "msgstream_obj_num",
			Help:      "number of message streams",
		})

	// RootCoordNumOfCredentials counts the number of credentials.
	RootCoordNumOfCredentials = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "credential_num",
			Help:      "number of credentials",
		})

	// RootCoordNumOfRoles counts the number of credentials.
	RootCoordNumOfRoles = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "num_of_roles",
			Help:      "The number of roles",
		})

	// RootCoordNumOfPrivilegeGroups counts the number of credentials.
	RootCoordNumOfPrivilegeGroups = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "num_of_privilege_groups",
			Help:      "The number of privilege groups",
		})

	// RootCoordTtDelay records the max time tick delay of flow graphs in DataNodes and QueryNodes.
	RootCoordTtDelay = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "time_tick_delay",
			Help:      "The max time tick delay of flow graphs",
		}, []string{
			roleNameLabelName,
			nodeIDLabelName,
		})

	// RootCoordQuotaStates records the quota states of cluster.
	RootCoordQuotaStates = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "quota_states",
			Help:      "The quota states of cluster",
		}, []string{
			"quota_states",
			"name",
		})

	// RootCoordForceDenyWritingCounter records the number of times that milvus turns into force-deny-writing states.
	RootCoordForceDenyWritingCounter = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "force_deny_writing_counter",
			Help:      "The number of times milvus turns into force-deny-writing states",
		})

	// RootCoordRateLimitRatio reflects the ratio of rate limit.
	RootCoordRateLimitRatio = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "rate_limit_ratio",
			Help:      "",
		}, []string{collectionIDLabelName})

	RootCoordDDLReqLatencyInQueue = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "ddl_req_latency_in_queue",
			Help:      "latency of each DDL operations in queue",
		}, []string{functionLabelName})

	RootCoordNumEntities = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "entity_num",
			Help:      "number of entities, clustered by collection and their status(loaded/total)",
		}, []string{
			databaseLabelName,
			collectionName,
			statusLabelName,
		})

	RootCoordIndexedNumEntities = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "indexed_entity_num",
			Help:      "indexed number of entities, clustered by collection, index name and whether it's a vector index",
		}, []string{
			databaseLabelName,
			collectionName,
			indexName,
			isVectorIndex,
		})

	QueryNodeMemoryHighWaterLevel = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "qn_mem_high_water_level",
			Help:      "querynode memory high water level",
		})

	DiskQuota = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: milvusNamespace,
			Subsystem: typeutil.RootCoordRole,
			Name:      "disk_quota",
			Help:      "disk quota",
		}, []string{"node_id", "scope"})
)

var (
	StreamingServiceClientRegisterOnce sync.Once

	// Streaming Service Client Producer Metrics.
	StreamingServiceClientResumingProducerTotal = newStreamingServiceClientGaugeVec(prometheus.GaugeOpts{
		Name: "resuming_producer_total",
		Help: "Total of resuming producers",
	}, WALChannelLabelName, StatusLabelName)

	StreamingServiceClientProducerTotal = newStreamingServiceClientGaugeVec(prometheus.GaugeOpts{
		Name: "producer_total",
		Help: "Total of producers",
	}, WALChannelLabelName, WALAccessModelLabelName)

	StreamingServiceClientProduceTotal = newStreamingServiceClientCounterVec(prometheus.CounterOpts{
		Name: "produce_total",
		Help: "Total of produce message",
	}, WALChannelLabelName, WALAccessModelLabelName, StatusLabelName)

	StreamingServiceClientProduceBytes = newStreamingServiceClientCounterVec(prometheus.CounterOpts{
		Name: "produce_bytes",
		Help: "Total of produce message",
	}, WALChannelLabelName, WALAccessModelLabelName, StatusLabelName)

	StreamingServiceClientSuccessProduceBytes = newStreamingServiceClientHistogramVec(prometheus.HistogramOpts{
		Name:    "produce_success_bytes",
		Help:    "Bytes of produced message",
		Buckets: messageBytesBuckets,
	}, WALChannelLabelName, WALAccessModelLabelName)

	StreamingServiceClientSuccessProduceDurationSeconds = newStreamingServiceClientHistogramVec(prometheus.HistogramOpts{
		Name:    "produce_success_duration_seconds",
		Help:    "Duration of produced message",
		Buckets: secondsBuckets,
	}, WALChannelLabelName, WALAccessModelLabelName)

	// Streaming Service Client Consumer Metrics.
	StreamingServiceClientResumingConsumerTotal = newStreamingServiceClientGaugeVec(prometheus.GaugeOpts{
		Name: "resuming_consumer_total",
		Help: "Total of resuming consumers",
	}, WALChannelLabelName, StatusLabelName)

	StreamingServiceClientConsumerTotal = newStreamingServiceClientGaugeVec(prometheus.GaugeOpts{
		Name: "consumer_total",
		Help: "Total of consumers",
	}, WALChannelLabelName, WALAccessModelLabelName)

	StreamingServiceClientConsumeBytes = newStreamingServiceClientHistogramVec(prometheus.HistogramOpts{
		Name:    "consume_bytes",
		Help:    "Bytes of consumed message",
		Buckets: messageBytesBuckets,
	}, WALChannelLabelName)

	// StreamingCoord metrics
	StreamingCoordPChannelInfo = newStreamingCoordGaugeVec(prometheus.GaugeOpts{
		Name: "pchannel_info",
		Help: "Term of pchannels",
	}, WALChannelLabelName, WALChannelTermLabelName, StreamingNodeLabelName, WALStateLabelName)

	StreamingCoordVChannelTotal = newStreamingCoordGaugeVec(prometheus.GaugeOpts{
		Name: "vchannel_total",
		Help: "Total of vchannels",
	}, WALChannelLabelName, StreamingNodeLabelName)

	StreamingCoordAssignmentVersion = newStreamingCoordGaugeVec(prometheus.GaugeOpts{
		Name: "assignment_info",
		Help: "Info of assignment",
	})

	StreamingCoordAssignmentListenerTotal = newStreamingCoordGaugeVec(prometheus.GaugeOpts{
		Name: "assignment_listener_total",
		Help: "Total of assignment listener",
	})

	StreamingCoordBroadcasterTaskTotal = newStreamingCoordGaugeVec(prometheus.GaugeOpts{
		Name: "broadcaster_task_total",
		Help: "Total of broadcaster task",
	}, BroadcasterTaskStateLabelName)

	StreamingCoordBroadcastDurationSeconds = newStreamingCoordHistogramVec(prometheus.HistogramOpts{
		Name:    "broadcaster_broadcast_duration_seconds",
		Help:    "Duration of broadcast",
		Buckets: secondsBuckets,
	})

	StreamingCoordBroadcasterAckAllDurationSeconds = newStreamingCoordHistogramVec(prometheus.HistogramOpts{
		Name:    "broadcaster_ack_all_duration_seconds",
		Help:    "Duration of acknowledge all message",
		Buckets: secondsBuckets,
	})

	StreamingCoordResourceKeyTotal = newStreamingCoordGaugeVec(prometheus.GaugeOpts{
		Name: "resource_key_total",
		Help: "Total of resource key hold at streaming coord",
	}, ResourceKeyDomainLabelName)

	// StreamingNode Producer Server Metrics.
	StreamingNodeProducerTotal = newStreamingNodeGaugeVec(prometheus.GaugeOpts{
		Name: "producer_total",
		Help: "Total of producers on current streaming node",
	}, WALChannelLabelName)

	StreamingNodeProduceInflightTotal = newStreamingNodeGaugeVec(prometheus.GaugeOpts{
		Name: "produce_inflight_total",
		Help: "Total of inflight produce request",
	}, WALChannelLabelName)

	// StreamingNode Consumer Server Metrics.
	StreamingNodeConsumerTotal = newStreamingNodeGaugeVec(prometheus.GaugeOpts{
		Name: "consumer_total",
		Help: "Total of consumers on current streaming node",
	}, WALChannelLabelName)

	StreamingNodeConsumeInflightTotal = newStreamingNodeGaugeVec(prometheus.GaugeOpts{
		Name: "consume_inflight_total",
		Help: "Total of inflight consume body",
	}, WALChannelLabelName)

	StreamingNodeConsumeBytes = newStreamingNodeHistogramVec(prometheus.HistogramOpts{
		Name:    "consume_bytes",
		Help:    "Bytes of consumed message",
		Buckets: messageBytesBuckets,
	}, WALChannelLabelName)

	// WAL WAL metrics
	WALInfo = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "info",
		Help: "current info of wal on current streaming node",
	}, WALChannelLabelName, WALChannelTermLabelName, WALNameLabelName)

	// TimeTick related metrics
	WALLastAllocatedTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "last_allocated_time_tick",
		Help: "Current max allocated time tick of wal",
	}, WALChannelLabelName)

	WALAllocateTimeTickTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "allocate_time_tick_total",
		Help: "Total of allocated time tick on wal",
	}, WALChannelLabelName, StatusLabelName)

	WALTimeTickAllocateDurationSeconds = newWALHistogramVec(prometheus.HistogramOpts{
		Name: "allocate_time_tick_duration_seconds",
		Help: "Duration of wal allocate time tick",
	}, WALChannelLabelName)

	WALLastConfirmedTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "last_confirmed_time_tick",
		Help: "Current max confirmed time tick of wal",
	}, WALChannelLabelName)

	WALAcknowledgeTimeTickTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "acknowledge_time_tick_total",
		Help: "Total of acknowledge time tick on wal",
	}, WALChannelLabelName, TimeTickAckTypeLabelName)

	WALSyncTimeTickTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "sync_time_tick_total",
		Help: "Total of sync time tick on wal",
	}, WALChannelLabelName, TimeTickAckTypeLabelName)

	WALTimeTickSyncTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "sync_total",
		Help: "Total of time tick sync sent",
	}, WALChannelLabelName, TimeTickSyncTypeLabelName)

	WALTimeTickSyncTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "sync_time_tick",
		Help: "Max time tick of time tick sync sent",
	}, WALChannelLabelName, TimeTickSyncTypeLabelName)

	// Txn Related Metrics
	WALInflightTxn = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "inflight_txn",
		Help: "Total of inflight txn on wal",
	}, WALChannelLabelName)

	WALTxnDurationSeconds = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "txn_duration_seconds",
		Help:    "Duration of wal txn",
		Buckets: secondsBuckets,
	}, WALChannelLabelName, WALTxnStateLabelName)

	// Rows level counter.
	WALInsertRowsTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "insert_rows_total",
		Help: "Rows of growing insert on wal",
	}, WALChannelLabelName)

	WALInsertBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "insert_bytes",
		Help: "Bytes of growing insert on wal",
	}, WALChannelLabelName)

	WALDeleteRowsTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "delete_rows_total",
		Help: "Rows of growing delete on wal",
	}, WALChannelLabelName)

	// Segment related metrics
	WALGrowingSegmentRowsTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "growing_segment_rows_total",
		Help: "Rows of segment growing on wal",
	}, WALChannelLabelName, WALSegmentLevelLabelName)

	WALGrowingSegmentBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "growing_segment_bytes",
		Help: "Bytes of segment growing on wal",
	}, WALChannelLabelName, WALSegmentLevelLabelName)

	WALGrowingSegmentHWMBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "growing_segment_hwm_bytes",
		Help: "HWM of segment growing bytes on node",
	})

	WALGrowingSegmentLWMBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "growing_segment_lwm_bytes",
		Help: "LWM of segment growing bytes on node",
	})

	WALSegmentAllocTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "segment_assign_segment_alloc_total",
		Help: "Total of segment alloc on wal",
	}, WALChannelLabelName)

	WALSegmentFlushedTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "segment_assign_flushed_segment_total",
		Help: "Total of segment sealed on wal",
	}, WALChannelLabelName, WALSegmentSealPolicyNameLabelName)

	WALSegmentRowsTotal = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "segment_assign_segment_rows_total",
		Help:    "Total rows of segment alloc on wal",
		Buckets: prometheus.ExponentialBucketsRange(128, 1048576, 10),
	}, WALChannelLabelName)

	WALSegmentBytes = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "segment_assign_segment_bytes",
		Help:    "Bytes of segment alloc on wal",
		Buckets: prometheus.ExponentialBucketsRange(5242880, 1073741824, 10),
	}, WALChannelLabelName)

	WALPartitionTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "segment_assign_partition_total",
		Help: "Total of partition on wal",
	}, WALChannelLabelName)

	WALCollectionTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "segment_assign_collection_total",
		Help: "Total of collection on wal",
	}, WALChannelLabelName)

	// Append Related Metrics
	WALAppendMessageBytes = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "append_message_bytes",
		Help:    "Bytes of append message to wal",
		Buckets: messageBytesBuckets,
	}, WALChannelLabelName, StatusLabelName)

	WALAppendMessageTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "append_message_total",
		Help: "Total of append message to wal",
	}, WALChannelLabelName, WALMessageTypeLabelName, StatusLabelName)

	WALAppendMessageBeforeInterceptorDurationSeconds = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "interceptor_before_append_duration_seconds",
		Help:    "Intercept duration before wal append message",
		Buckets: secondsBuckets,
	}, WALChannelLabelName, WALInterceptorLabelName)

	WALAppendMessageAfterInterceptorDurationSeconds = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "interceptor_after_append_duration_seconds",
		Help:    "Intercept duration after wal append message",
		Buckets: secondsBuckets,
	}, WALChannelLabelName, WALInterceptorLabelName)

	WALImplsAppendRetryTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "impls_append_message_retry_total",
		Help: "Total of append message retry",
	}, WALChannelLabelName)

	WALAppendMessageDurationSeconds = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "append_message_duration_seconds",
		Help:    "Duration of wal append message",
		Buckets: secondsBuckets,
	}, WALChannelLabelName, StatusLabelName)

	WALImplsAppendMessageDurationSeconds = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "impls_append_message_duration_seconds",
		Help:    "Duration of wal impls append message",
		Buckets: secondsBuckets,
	}, WALChannelLabelName, StatusLabelName)

	WALWriteAheadBufferEntryTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "write_ahead_buffer_entry_total",
		Help: "Total of write ahead buffer entry in wal",
	}, WALChannelLabelName)

	WALWriteAheadBufferSizeBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "write_ahead_buffer_size_bytes",
		Help: "Size of write ahead buffer in wal",
	}, WALChannelLabelName)

	WALWriteAheadBufferCapacityBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "write_ahead_buffer_capacity_bytes",
		Help: "Capacity of write ahead buffer in wal",
	}, WALChannelLabelName)

	WALWriteAheadBufferEarliestTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "write_ahead_buffer_earliest_time_tick",
		Help: "Earliest time tick of write ahead buffer in wal",
	}, WALChannelLabelName)

	WALWriteAheadBufferLatestTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "write_ahead_buffer_latest_time_tick",
		Help: "Latest time tick of write ahead buffer in wal",
	}, WALChannelLabelName)

	// Scanner Related Metrics
	WALScannerTotal = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "scanner_total",
		Help: "Total of wal scanner on current streaming node",
	}, WALChannelLabelName, WALScannerModelLabelName)

	WALScanMessageBytes = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "scan_message_bytes",
		Help:    "Bytes of scanned message from wal",
		Buckets: messageBytesBuckets,
	}, WALChannelLabelName, WALScannerModelLabelName)

	WALScanMessageTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "scan_message_total",
		Help: "Total of scanned message from wal",
	}, WALChannelLabelName, WALMessageTypeLabelName, WALScannerModelLabelName)

	WALScanPassMessageBytes = newWALHistogramVec(prometheus.HistogramOpts{
		Name:    "scan_pass_message_bytes",
		Help:    "Bytes of pass (not filtered) scanned message from wal",
		Buckets: messageBytesBuckets,
	}, WALChannelLabelName, WALScannerModelLabelName)

	WALScanPassMessageTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "scan_pass_message_total",
		Help: "Total of pass (not filtered) scanned message from wal",
	}, WALChannelLabelName, WALMessageTypeLabelName, WALScannerModelLabelName)

	WALScanTimeTickViolationMessageTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "scan_time_tick_violation_message_total",
		Help: "Total of time tick violation message (dropped) from wal",
	}, WALChannelLabelName, WALMessageTypeLabelName, WALScannerModelLabelName)

	WALScanTxnTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "scan_txn_total",
		Help: "Total of scanned txn from wal",
	}, WALChannelLabelName, WALTxnStateLabelName)

	WALScannerPendingQueueBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "scanner_pending_queue_bytes",
		Help: "Size of pending queue in wal scanner",
	}, WALChannelLabelName)

	WALScannerTimeTickBufBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "scanner_time_tick_buf_bytes",
		Help: "Size of time tick buffer in wal scanner",
	}, WALChannelLabelName)

	WALScannerTxnBufBytes = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "scanner_txn_buf_bytes",
		Help: "Size of txn buffer in wal scanner",
	}, WALChannelLabelName)

	WALFlusherInfo = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "flusher_info",
		Help: "Current info of flusher on current wal",
	}, WALChannelLabelName, WALChannelTermLabelName, WALFlusherStateLabelName)

	WALFlusherTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "flusher_time_tick",
		Help: "the final timetick tick of flusher seen",
	}, WALChannelLabelName, WALChannelTermLabelName)

	WALRecoveryInfo = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "recovery_info",
		Help: "Current info of recovery storage on current wal",
	}, WALChannelLabelName, WALChannelTermLabelName, WALRecoveryStorageStateLabelName)

	WALRecoveryInMemTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "recovery_in_mem_time_tick",
		Help: "the final timetick tick of recovery storage seen",
	}, WALChannelLabelName, WALChannelTermLabelName)

	WALRecoveryPersistedTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "recovery_persisted_time_tick",
		Help: "the final persisted timetick tick of recovery storage seen",
	}, WALChannelLabelName, WALChannelTermLabelName)

	WALRecoveryInconsistentEventTotal = newWALCounterVec(prometheus.CounterOpts{
		Name: "recovery_inconsistent_event_total",
		Help: "Total of recovery inconsistent event",
	}, WALChannelLabelName, WALChannelTermLabelName)

	WALRecoveryIsOnPersisting = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "recovery_is_on_persisting",
		Help: "Is recovery storage on persisting",
	}, WALChannelLabelName, WALChannelTermLabelName)

	WALTruncateTimeTick = newWALGaugeVec(prometheus.GaugeOpts{
		Name: "truncate_time_tick",
		Help: "the final timetick tick of truncator seen",
	}, WALChannelLabelName, WALChannelTermLabelName)
)

Functions

func CleanQueryCoordMetricsWithCollectionID

func CleanQueryCoordMetricsWithCollectionID(collectionID int64)

func CleanupDataCoordWithCollectionID

func CleanupDataCoordWithCollectionID(collectionID int64)

func CleanupDataNodeCollectionMetrics

func CleanupDataNodeCollectionMetrics(nodeID int64, collectionID int64, channel string)

func CleanupProxyCollectionMetrics

func CleanupProxyCollectionMetrics(nodeID int64, dbName string, collection string)

func CleanupProxyDBMetrics

func CleanupProxyDBMetrics(nodeID int64, dbName string)

func CleanupQueryNodeCollectionMetrics

func CleanupQueryNodeCollectionMetrics(nodeID int64, collectionID int64)

func CleanupRootCoordDBMetrics

func CleanupRootCoordDBMetrics(dbName string)

func GetRegisterer

func GetRegisterer() prometheus.Registerer

GetRegisterer returns the global prometheus registerer metricsRegistry must be call after Register is called or no Register is called.

func NewGRPCSizeStatsHandler

func NewGRPCSizeStatsHandler() *grpcSizeStatsHandler

func RecordRestfulMetrics

func RecordRestfulMetrics(ctx context.Context, outputLength int64, observeOutbound bool)

func Register

func Register(r prometheus.Registerer)

Register serves prometheus http service Should be called by init function.

func RegisterCGOMetrics

func RegisterCGOMetrics(registry *prometheus.Registry)

RegisterCGOMetrics registers the cgo metrics.

func RegisterDataCoord

func RegisterDataCoord(registry *prometheus.Registry)

RegisterDataCoord registers DataCoord metrics

func RegisterDataNode

func RegisterDataNode(registry *prometheus.Registry)

RegisterDataNode registers DataNode metrics

func RegisterMQType

func RegisterMQType(mq string)

RegisterMQType registers the type of mq

func RegisterMetaMetrics

func RegisterMetaMetrics(registry *prometheus.Registry)

RegisterMetaMetrics registers meta metrics

func RegisterMetaType

func RegisterMetaType(meta string)

RegisterMetaType registers the type of meta

func RegisterMixCoord

func RegisterMixCoord(registry *prometheus.Registry)

RegisterRootCoord registers RootCoord metrics

func RegisterMsgStreamMetrics

func RegisterMsgStreamMetrics(registry *prometheus.Registry)

RegisterMsgStreamMetrics registers msg stream metrics

func RegisterProxy

func RegisterProxy(registry *prometheus.Registry)

RegisterProxy registers Proxy metrics

func RegisterQueryCoord

func RegisterQueryCoord(registry *prometheus.Registry)

RegisterQueryCoord registers QueryCoord metrics

func RegisterQueryNode

func RegisterQueryNode(registry *prometheus.Registry)

RegisterQueryNode registers QueryNode metrics

func RegisterStorageMetrics

func RegisterStorageMetrics(registry *prometheus.Registry)

RegisterStorageMetrics registers storage metrics

func RegisterStreamingNode

func RegisterStreamingNode(registry *prometheus.Registry)

RegisterStreamingNode registers streaming node metrics

func RegisterStreamingServiceClient

func RegisterStreamingServiceClient(registry *prometheus.Registry)

RegisterStreamingServiceClient registers streaming service client metrics

func WrapRestfulContext

func WrapRestfulContext(ctx context.Context, inputLength int64) context.Context

Types

type RPCStats

type RPCStats struct {
	// contains filtered or unexported fields
}

RPCStats stores the meta and payload size info it should be attached to context so that request sizing could be avoided

func GetStats

func GetStats(ctx context.Context) *RPCStats

func (*RPCStats) SetCollectionName

func (s *RPCStats) SetCollectionName(collName string) *RPCStats

func (*RPCStats) SetDatabaseName

func (s *RPCStats) SetDatabaseName(collName string) *RPCStats

func (*RPCStats) SetInboundLabel

func (s *RPCStats) SetInboundLabel(label string) *RPCStats

func (*RPCStats) SetNodeID

func (s *RPCStats) SetNodeID(nodeID int64) *RPCStats