config

package
v1.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 21, 2021 License: Apache-2.0 Imports: 36 Imported by: 10

Documentation

Index

Constants

View Source 
const (
	// DefaultNewFileMode is the default new file mode.
	DefaultNewFileMode = os.FileMode(0666)
	// DefaultNewDirectoryMode is the default new directory mode.
	DefaultNewDirectoryMode = os.FileMode(0755)
)

Variables

This section is empty.

Functions

func InitialClusterEndpoints 

func InitialClusterEndpoints(initialCluster []environment.SeedNode) ([]string, error)

InitialClusterEndpoints returns the endpoints of the initial cluster

func IsSeedNode 

func IsSeedNode(initialCluster []environment.SeedNode, hostID string) bool

IsSeedNode returns whether the given hostID is an etcd node.

func NewEtcdEmbedConfig 

func NewEtcdEmbedConfig(cfg DBConfiguration) (*embed.Config, error)

NewEtcdEmbedConfig creates a new embedded etcd config from kv config.

Types

type BlockRetrievePolicy 

type BlockRetrievePolicy struct {
	// FetchConcurrency is the concurrency to fetch blocks from disk. For
	// spinning disks it is highly recommended to set this value to 1.
	FetchConcurrency *int `yaml:"fetchConcurrency" validate:"min=1"`

	// CacheBlocksOnRetrieve globally enables/disables callbacks used to cache blocks fetched
	// from disk.
	CacheBlocksOnRetrieve *bool `yaml:"cacheBlocksOnRetrieve"`
}

BlockRetrievePolicy is the block retrieve policy.

type BootstrapCommitlogConfiguration added in v0.4.8 

type BootstrapCommitlogConfiguration struct {
	// ReturnUnfulfilledForCorruptCommitLogFiles controls whether the commitlog bootstrapper
	// will return unfulfilled for all shard time ranges when it encounters a corrupt commit
	// file. Note that regardless of this value, the commitlog bootstrapper will still try and
	// read all the uncorrupted commitlog files and return as much data as it can, but setting
	// this to true allows the node to attempt a repair if the peers bootstrapper is configured
	// after the commitlog bootstrapper.
	ReturnUnfulfilledForCorruptCommitLogFiles bool `yaml:"returnUnfulfilledForCorruptCommitLogFiles"`
}

BootstrapCommitlogConfiguration specifies config for the commitlog bootstrapper.

type BootstrapConfiguration 

type BootstrapConfiguration struct {
	// BootstrapMode defines the mode in which bootstrappers are run.
	BootstrapMode *BootstrapMode `yaml:"mode"`

	// Filesystem bootstrapper configuration.
	Filesystem *BootstrapFilesystemConfiguration `yaml:"filesystem"`

	// Commitlog bootstrapper configuration.
	Commitlog *BootstrapCommitlogConfiguration `yaml:"commitlog"`

	// Peers bootstrapper configuration.
	Peers *BootstrapPeersConfiguration `yaml:"peers"`

	// CacheSeriesMetadata determines whether individual bootstrappers cache
	// series metadata across all calls (namespaces / shards / blocks).
	CacheSeriesMetadata *bool `yaml:"cacheSeriesMetadata"`

	// IndexSegmentConcurrency determines the concurrency for building index
	// segments.
	IndexSegmentConcurrency *int `yaml:"indexSegmentConcurrency"`

	// Verify specifies verification checks.
	Verify *BootstrapVerifyConfiguration `yaml:"verify"`
}

BootstrapConfiguration specifies the config for bootstrappers.

func (BootstrapConfiguration) New 

func (bsc BootstrapConfiguration) New(
	rsOpts result.Options,
	opts storage.Options,
	topoMapProvider topology.MapProvider,
	origin topology.Host,
	adminClient client.AdminClient,
) (bootstrap.ProcessProvider, error)

New creates a bootstrap process based on the bootstrap configuration.

func (BootstrapConfiguration) VerifyOrDefault added in v1.0.0 

func (bsc BootstrapConfiguration) VerifyOrDefault() BootstrapVerifyConfiguration

VerifyOrDefault returns verify configuration or default.

type BootstrapFilesystemConfiguration 

type BootstrapFilesystemConfiguration struct {
	// DeprecatedNumProcessorsPerCPU is the number of processors per CPU.
	// TODO: Remove, this is deprecated since BootstrapDataNumProcessors() is
	// no longer actually used anywhere.
	DeprecatedNumProcessorsPerCPU float64 `yaml:"numProcessorsPerCPU" validate:"min=0.0"`

	// Migration configuration specifies what version, if any, existing data filesets should be migrated to
	// if necessary.
	Migration *BootstrapMigrationConfiguration `yaml:"migration"`
}

BootstrapFilesystemConfiguration specifies config for the fs bootstrapper.

type BootstrapMigrationConfiguration added in v0.15.9 

type BootstrapMigrationConfiguration struct {
	// TargetMigrationVersion indicates that we should attempt to upgrade filesets to
	// what’s expected of the specified version.
	TargetMigrationVersion migration.MigrationVersion `yaml:"targetMigrationVersion"`

	// Concurrency sets the number of concurrent workers performing migrations.
	Concurrency int `yaml:"concurrency"`
}

BootstrapMigrationConfiguration specifies configuration for data migrations during bootstrapping.

func (BootstrapMigrationConfiguration) NewOptions added in v0.15.9 

func (m BootstrapMigrationConfiguration) NewOptions() migration.Options

NewOptions generates migration.Options from the configuration.

type BootstrapMode added in v1.0.0 

type BootstrapMode uint

BootstrapMode defines the mode in which bootstrappers are run. 

const (
	// DefaultBootstrapMode executes bootstrappers in default order.
	DefaultBootstrapMode BootstrapMode = iota
	// PreferPeersBootstrapMode executes peers before commitlog bootstrapper.
	PreferPeersBootstrapMode
	// ExcludeCommitLogBootstrapMode executes all default bootstrappers except commitlog.
	ExcludeCommitLogBootstrapMode
)

func (BootstrapMode) String added in v1.0.0 

func (m BootstrapMode) String() string

String returns the bootstrap mode as a string

func (*BootstrapMode) UnmarshalYAML added in v1.0.0 

func (m *BootstrapMode) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML unmarshals an BootstrapMode into a valid type from string.

type BootstrapPeersConfiguration 

type BootstrapPeersConfiguration struct {
	// StreamShardConcurrency controls how many shards in parallel to stream
	// for in memory data being streamed between peers (most recent block).
	// Defaults to: numCPU.
	StreamShardConcurrency int `yaml:"streamShardConcurrency"`
	// StreamPersistShardConcurrency controls how many shards in parallel to stream
	// for historical data being streamed between peers (historical blocks).
	// Defaults to: numCPU / 2.
	StreamPersistShardConcurrency int `yaml:"streamPersistShardConcurrency"`
	// StreamPersistShardFlushConcurrency controls how many shards in parallel to flush
	// for historical data being streamed between peers (historical blocks).
	// Defaults to: 1.
	StreamPersistShardFlushConcurrency int `yaml:"streamPersistShardFlushConcurrency"`
}

BootstrapPeersConfiguration specifies config for the peers bootstrapper.

type BootstrapVerifyConfiguration added in v1.0.0 

type BootstrapVerifyConfiguration struct {
	VerifyIndexSegments *bool `yaml:"verifyIndexSegments"`
}

BootstrapVerifyConfiguration outlines verification checks to enable during a bootstrap.

func (BootstrapVerifyConfiguration) VerifyIndexSegmentsOrDefault added in v1.0.0 

func (c BootstrapVerifyConfiguration) VerifyIndexSegmentsOrDefault() bool

VerifyIndexSegmentsOrDefault returns whether to verify index segments or use default value.

type BucketPoolPolicy 

type BucketPoolPolicy struct {
	// The pool buckets sizes to use
	Buckets []CapacityPoolPolicy `yaml:"buckets"`
}

BucketPoolPolicy specifies a bucket pool policy.

type CacheConfigurations 

type CacheConfigurations struct {
	// Series cache policy.
	Series *SeriesCacheConfiguration `yaml:"series"`

	// PostingsList cache policy.
	PostingsList *PostingsListCacheConfiguration `yaml:"postingsList"`

	// Regexp cache policy.
	Regexp *RegexpCacheConfiguration `yaml:"regexp"`
}

CacheConfigurations is the cache configurations.

func (CacheConfigurations) PostingsListConfiguration added in v0.6.0 

func (c CacheConfigurations) PostingsListConfiguration() PostingsListCacheConfiguration

PostingsListConfiguration returns the postings list cache configuration or default if none is specified.

func (CacheConfigurations) RegexpConfiguration added in v1.0.0 

func (c CacheConfigurations) RegexpConfiguration() RegexpCacheConfiguration

RegexpConfiguration returns the regexp cache configuration or default if none is specified.

func (CacheConfigurations) SeriesConfiguration 

func (c CacheConfigurations) SeriesConfiguration() SeriesCacheConfiguration

SeriesConfiguration returns the series cache configuration or default if none is specified.

type CalculationType 

type CalculationType string

CalculationType is a type of configuration parameter. 

const (
	// CalculationTypeFixed is a fixed parameter not to be scaled of any parameter.
	CalculationTypeFixed CalculationType = "fixed"
	// CalculationTypePerCPU is a parameter that needs to be scaled by number of CPUs.
	CalculationTypePerCPU CalculationType = "percpu"
)

type CapacityPoolPolicy 

type CapacityPoolPolicy struct {
	PoolPolicy `yaml:",inline"`

	// The capacity of items in the pool.
	Capacity *int `yaml:"capacity"`
}

CapacityPoolPolicy specifies a single pool policy that has a per element capacity.

func (*CapacityPoolPolicy) CapacityOrDefault added in v0.6.0 

func (p *CapacityPoolPolicy) CapacityOrDefault() int

CapacityOrDefault returns the configured capacity if present, or a default value otherwise.

type CommitLogPolicy 

type CommitLogPolicy struct {
	// The max size the commit log will flush a segment to disk after buffering.
	FlushMaxBytes int `yaml:"flushMaxBytes" validate:"nonzero"`

	// The maximum amount of time the commit log will wait to flush to disk.
	FlushEvery time.Duration `yaml:"flushEvery" validate:"nonzero"`

	// The queue the commit log will keep in front of the current commit log segment.
	// Modifying values in this policy will control how many pending writes can be
	// in the commitlog queue before M3DB will begin rejecting writes.
	Queue CommitLogQueuePolicy `yaml:"queue" validate:"nonzero"`

	// The actual Golang channel that implements the commit log queue. We separate this
	// from the Queue field for historical / legacy reasons. Generally speaking, the
	// values in this config should not need to be modified, but we leave it in for
	// tuning purposes. Unlike the Queue field, values in this policy control the size
	// of the channel that backs the queue. Since writes to the commitlog are batched,
	// setting the size of this policy will control how many batches can be queued, and
	// indrectly how many writes can be queued, but that is dependent on the batch size
	// of the client. As a result, we recommend that users avoid tuning this field and
	// modify the Queue size instead which maps directly to the number of writes. This
	// works in most cases because the default size of the QueueChannel should be large
	// enough for almost all workloads assuming a reasonable batch size is used.
	QueueChannel *CommitLogQueuePolicy `yaml:"queueChannel"`
}

CommitLogPolicy is the commit log policy.

type CommitLogQueuePolicy 

type CommitLogQueuePolicy struct {
	// The type of calculation for the size.
	CalculationType CalculationType `yaml:"calculationType"`

	// The size of the commit log, calculated according to the calculation type.
	Size int `yaml:"size" validate:"nonzero"`
}

CommitLogQueuePolicy is the commit log queue policy.

type Configuration 

type Configuration struct {
	// DB is the configuration for a DB node (required).
	DB *DBConfiguration `yaml:"db"`

	// Coordinator is the configuration for the coordinator to run (optional).
	Coordinator *coordinatorcfg.Configuration `yaml:"coordinator"`
}

Configuration is the top level configuration that includes both a DB node and a coordinator.

func (*Configuration) Validate added in v1.0.0 

func (c *Configuration) Validate() error

Validate validates the Configuration. We use this method to validate fields where the validator package falls short.

type DBConfiguration 

type DBConfiguration struct {
	// Index configuration.
	Index IndexConfiguration `yaml:"index"`

	// Transforms configuration.
	Transforms TransformConfiguration `yaml:"transforms"`

	// Logging configuration.
	Logging *xlog.Configuration `yaml:"logging"`

	// Metrics configuration.
	Metrics *instrument.MetricsConfiguration `yaml:"metrics"`

	// The host and port on which to listen for the node service.
	ListenAddress *string `yaml:"listenAddress"`

	// The host and port on which to listen for the cluster service.
	ClusterListenAddress *string `yaml:"clusterListenAddress"`

	// The HTTP host and port on which to listen for the node service.
	HTTPNodeListenAddress *string `yaml:"httpNodeListenAddress"`

	// The HTTP host and port on which to listen for the cluster service.
	HTTPClusterListenAddress *string `yaml:"httpClusterListenAddress"`

	// The host and port on which to listen for debug endpoints.
	DebugListenAddress *string `yaml:"debugListenAddress"`

	// HostID is the local host ID configuration.
	HostID *hostid.Configuration `yaml:"hostID"`

	// Client configuration, used for inter-node communication and when used as a coordinator.
	Client client.Configuration `yaml:"client"`

	// The initial garbage collection target percentage.
	GCPercentage int `yaml:"gcPercentage" validate:"max=100"`

	// The tick configuration, omit this to use default settings.
	Tick *TickConfiguration `yaml:"tick"`

	// Bootstrap configuration.
	Bootstrap BootstrapConfiguration `yaml:"bootstrap"`

	// The block retriever policy.
	BlockRetrieve *BlockRetrievePolicy `yaml:"blockRetrieve"`

	// Cache configurations.
	Cache CacheConfigurations `yaml:"cache"`

	// The filesystem configuration for the node.
	Filesystem FilesystemConfiguration `yaml:"filesystem"`

	// The commit log policy for the node.
	CommitLog *CommitLogPolicy `yaml:"commitlog"`

	// The repair policy for repairing data within a cluster.
	Repair *RepairPolicy `yaml:"repair"`

	// The replication policy for replicating data between clusters.
	Replication *ReplicationPolicy `yaml:"replication"`

	// The pooling policy.
	PoolingPolicy *PoolingPolicy `yaml:"pooling"`

	// The discovery configuration.
	Discovery *discovery.Configuration `yaml:"discovery"`

	// The configuration for hashing
	Hashing HashingConfiguration `yaml:"hashing"`

	// Write new series asynchronously for fast ingestion of new ID bursts.
	WriteNewSeriesAsync *bool `yaml:"writeNewSeriesAsync"`

	// Write new series backoff between batches of new series insertions.
	WriteNewSeriesBackoffDuration *time.Duration `yaml:"writeNewSeriesBackoffDuration"`

	// Proto contains the configuration specific to running in the ProtoDataMode.
	Proto *ProtoConfiguration `yaml:"proto"`

	// Tracing configures opentracing. If not provided, tracing is disabled.
	Tracing *opentracing.TracingConfiguration `yaml:"tracing"`

	// Limits contains configuration for limits that can be applied to M3DB for the purposes
	// of applying back-pressure or protecting the db nodes.
	Limits LimitsConfiguration `yaml:"limits"`

	// WideConfig contains some limits for wide operations. These operations
	// differ from regular paths by optimizing for query completeness across
	// arbitary query ranges rather than speed.
	WideConfig *WideConfiguration `yaml:"wide"`

	// TChannel exposes TChannel config options.
	TChannel *TChannelConfiguration `yaml:"tchannel"`

	// Debug configuration.
	Debug config.DebugConfiguration `yaml:"debug"`

	// ForceColdWritesEnabled will force enable cold writes for all namespaces
	// if set.
	ForceColdWritesEnabled *bool `yaml:"forceColdWritesEnabled"`
}

DBConfiguration is the configuration for a DB node.

func (*DBConfiguration) ClusterListenAddressOrDefault added in v1.0.0 

func (c *DBConfiguration) ClusterListenAddressOrDefault() string

ClusterListenAddressOrDefault returns the listen address or default.

func (*DBConfiguration) CommitLogOrDefault added in v1.0.0 

func (c *DBConfiguration) CommitLogOrDefault() CommitLogPolicy

CommitLogOrDefault returns the commit log policy or default.

func (*DBConfiguration) DebugListenAddressOrDefault added in v1.0.0 

func (c *DBConfiguration) DebugListenAddressOrDefault() string

DebugListenAddressOrDefault returns the listen address or default.

func (*DBConfiguration) DiscoveryOrDefault added in v1.0.0 

func (c *DBConfiguration) DiscoveryOrDefault() discovery.Configuration

DiscoveryOrDefault returns the discovery configuration or defaults.

func (*DBConfiguration) GCPercentageOrDefault added in v1.0.0 

func (c *DBConfiguration) GCPercentageOrDefault() int

GCPercentageOrDefault returns the GC percentage or default.

func (*DBConfiguration) HTTPClusterListenAddressOrDefault added in v1.0.0 

func (c *DBConfiguration) HTTPClusterListenAddressOrDefault() string

HTTPClusterListenAddressOrDefault returns the listen address or default.

func (*DBConfiguration) HTTPNodeListenAddressOrDefault added in v1.0.0 

func (c *DBConfiguration) HTTPNodeListenAddressOrDefault() string

HTTPNodeListenAddressOrDefault returns the listen address or default.

func (*DBConfiguration) HostIDOrDefault added in v1.0.0 

func (c *DBConfiguration) HostIDOrDefault() hostid.Configuration

HostIDOrDefault returns the host ID or default.

func (*DBConfiguration) ListenAddressOrDefault added in v1.0.0 

func (c *DBConfiguration) ListenAddressOrDefault() string

ListenAddressOrDefault returns the listen address or default.

func (*DBConfiguration) LoggingOrDefault added in v1.0.0 

func (c *DBConfiguration) LoggingOrDefault() xlog.Configuration

LoggingOrDefault returns the logging configuration or defaults.

func (*DBConfiguration) MetricsOrDefault added in v1.0.0 

func (c *DBConfiguration) MetricsOrDefault() *instrument.MetricsConfiguration

MetricsOrDefault returns metrics configuration or defaults.

func (*DBConfiguration) PoolingPolicyOrDefault added in v1.0.0 

func (c *DBConfiguration) PoolingPolicyOrDefault() (PoolingPolicy, error)

PoolingPolicyOrDefault returns the pooling policy or default.

func (*DBConfiguration) Validate added in v1.0.0 

func (c *DBConfiguration) Validate() error

Validate validates the Configuration. We use this method to validate fields where the validator package falls short.

func (*DBConfiguration) WriteNewSeriesAsyncOrDefault added in v1.0.0 

func (c *DBConfiguration) WriteNewSeriesAsyncOrDefault() bool

WriteNewSeriesAsyncOrDefault returns whether to write new series async or not.

func (*DBConfiguration) WriteNewSeriesBackoffDurationOrDefault added in v1.0.0 

func (c *DBConfiguration) WriteNewSeriesBackoffDurationOrDefault() time.Duration

WriteNewSeriesBackoffDurationOrDefault returns the backoff duration for new series inserts.

type FilesystemConfiguration 

type FilesystemConfiguration struct {
	// File path prefix for reading/writing TSDB files
	FilePathPrefix *string `yaml:"filePathPrefix"`

	// Write buffer size
	WriteBufferSize *int `yaml:"writeBufferSize"`

	// Data read buffer size
	DataReadBufferSize *int `yaml:"dataReadBufferSize"`

	// Info metadata file read buffer size
	InfoReadBufferSize *int `yaml:"infoReadBufferSize"`

	// Seek data read buffer size
	SeekReadBufferSize *int `yaml:"seekReadBufferSize"`

	// Disk flush throughput limit in Mb/s
	ThroughputLimitMbps *float64 `yaml:"throughputLimitMbps"`

	// Disk flush throughput check interval
	ThroughputCheckEvery *int `yaml:"throughputCheckEvery"`

	// NewFileMode is the new file permissions mode to use when
	// creating files - specify as three digits, e.g. 666.
	NewFileMode *string `yaml:"newFileMode"`

	// NewDirectoryMode is the new file permissions mode to use when
	// creating directories - specify as three digits, e.g. 755.
	NewDirectoryMode *string `yaml:"newDirectoryMode"`

	// Mmap is the mmap options which features are primarily platform dependent
	Mmap *MmapConfiguration `yaml:"mmap"`

	// ForceIndexSummariesMmapMemory forces the mmap that stores the index lookup bytes
	// to be an anonymous region in memory as opposed to a file-based mmap.
	ForceIndexSummariesMmapMemory *bool `yaml:"force_index_summaries_mmap_memory"`

	// ForceBloomFilterMmapMemory forces the mmap that stores the index lookup bytes
	// to be an anonymous region in memory as opposed to a file-based mmap.
	ForceBloomFilterMmapMemory *bool `yaml:"force_bloom_filter_mmap_memory"`

	// BloomFilterFalsePositivePercent controls the target false positive percentage
	// for the bloom filters for the fileset files.
	BloomFilterFalsePositivePercent *float64 `yaml:"bloomFilterFalsePositivePercent"`
}

FilesystemConfiguration is the filesystem configuration.

func (FilesystemConfiguration) BloomFilterFalsePositivePercentOrDefault added in v0.14.0 

func (f FilesystemConfiguration) BloomFilterFalsePositivePercentOrDefault() float64

BloomFilterFalsePositivePercentOrDefault returns the configured value for the target false positive percent for the bloom filter for the fileset files if configured, or a default value otherwise

func (FilesystemConfiguration) DataReadBufferSizeOrDefault added in v0.6.0 

func (f FilesystemConfiguration) DataReadBufferSizeOrDefault() int

DataReadBufferSizeOrDefault returns the configured data read buffer size if configured, or a default value otherwise.

func (FilesystemConfiguration) FilePathPrefixOrDefault added in v0.6.0 

func (f FilesystemConfiguration) FilePathPrefixOrDefault() string

FilePathPrefixOrDefault returns the configured file path prefix if configured, or a default value otherwise.

func (FilesystemConfiguration) ForceBloomFilterMmapMemoryOrDefault added in v0.6.0 

func (f FilesystemConfiguration) ForceBloomFilterMmapMemoryOrDefault() bool

ForceBloomFilterMmapMemoryOrDefault returns the configured value for forcing the bloom filter mmaps into anonymous region in memory if configured, or a default value otherwise.

func (FilesystemConfiguration) ForceIndexSummariesMmapMemoryOrDefault added in v0.6.0 

func (f FilesystemConfiguration) ForceIndexSummariesMmapMemoryOrDefault() bool

ForceIndexSummariesMmapMemoryOrDefault returns the configured value for forcing the summaries mmaps into anonymous region in memory if configured, or a default value otherwise.

func (FilesystemConfiguration) InfoReadBufferSizeOrDefault added in v0.6.0 

func (f FilesystemConfiguration) InfoReadBufferSizeOrDefault() int

InfoReadBufferSizeOrDefault returns the configured info read buffer size if configured, or a default value otherwise.

func (FilesystemConfiguration) MmapConfigurationOrDefault added in v0.6.0 

func (f FilesystemConfiguration) MmapConfigurationOrDefault() MmapConfiguration

MmapConfigurationOrDefault returns the configured mmap configuration if configured, or a default value otherwise.

func (FilesystemConfiguration) ParseNewDirectoryMode 

func (f FilesystemConfiguration) ParseNewDirectoryMode() (os.FileMode, error)

ParseNewDirectoryMode parses the specified new directory mode.

func (FilesystemConfiguration) ParseNewFileMode 

func (f FilesystemConfiguration) ParseNewFileMode() (os.FileMode, error)

ParseNewFileMode parses the specified new file mode.

func (FilesystemConfiguration) SeekReadBufferSizeOrDefault added in v0.6.0 

func (f FilesystemConfiguration) SeekReadBufferSizeOrDefault() int

SeekReadBufferSizeOrDefault returns the configured seek read buffer size if configured, or a default value otherwise.

func (FilesystemConfiguration) ThroughputCheckEveryOrDefault added in v0.6.0 

func (f FilesystemConfiguration) ThroughputCheckEveryOrDefault() int

ThroughputCheckEveryOrDefault returns the configured throughput check every value if configured, or a default value otherwise.

func (FilesystemConfiguration) ThroughputLimitMbpsOrDefault added in v0.6.0 

func (f FilesystemConfiguration) ThroughputLimitMbpsOrDefault() float64

ThroughputLimitMbpsOrDefault returns the configured throughput limit mbps if configured, or a default value otherwise.

func (FilesystemConfiguration) Validate added in v0.6.0 

func (f FilesystemConfiguration) Validate() error

Validate validates the Filesystem configuration. We use this method to validate fields where the validator package falls short.

func (FilesystemConfiguration) WriteBufferSizeOrDefault added in v0.6.0 

func (f FilesystemConfiguration) WriteBufferSizeOrDefault() int

WriteBufferSizeOrDefault returns the configured write buffer size if configured, or a default value otherwise.

type HashingConfiguration 

type HashingConfiguration struct {
	// Murmur32 seed value.
	Seed uint32 `yaml:"seed"`
}

HashingConfiguration is the configuration for hashing.

type IndexConfiguration added in v0.3.0 

type IndexConfiguration struct {
	// MaxQueryIDsConcurrency controls the maximum number of outstanding QueryID
	// requests that can be serviced concurrently. Limiting the concurrency is
	// important to prevent index queries from overloading the database entirely
	// as they are very CPU-intensive (regex and FST matching).
	MaxQueryIDsConcurrency int `yaml:"maxQueryIDsConcurrency" validate:"min=0"`

	// RegexpDFALimit is the limit on the max number of states used by a
	// regexp deterministic finite automaton. Default is 10,000 states.
	RegexpDFALimit *int `yaml:"regexpDFALimit"`

	// RegexpFSALimit is the limit on the max number of bytes used by the
	// finite state automaton. Default is 10mb (10 million as int).
	RegexpFSALimit *uint `yaml:"regexpFSALimit"`

	// ForwardIndexProbability determines the likelihood that an incoming write is
	// written to the next block, when arriving close to the block boundary.
	//
	// NB: this is an optimization which lessens pressure on the index around
	// block boundaries by eagerly writing the series to the next block
	// preemptively.
	ForwardIndexProbability float64 `yaml:"forwardIndexProbability" validate:"min=0.0,max=1.0"`

	// ForwardIndexThreshold determines the threshold for forward writes, as a
	// fraction of the given namespace's bufferFuture.
	//
	// NB: this is an optimization which lessens pressure on the index around
	// block boundaries by eagerly writing the series to the next block
	// preemptively.
	ForwardIndexThreshold float64 `yaml:"forwardIndexThreshold" validate:"min=0.0,max=1.0"`
}

IndexConfiguration contains index-specific configuration.

func (IndexConfiguration) RegexpDFALimitOrDefault added in v1.0.1 

func (c IndexConfiguration) RegexpDFALimitOrDefault() int

RegexpDFALimitOrDefault returns the deterministic finite automaton states limit or default.

func (IndexConfiguration) RegexpFSALimitOrDefault added in v1.0.1 

func (c IndexConfiguration) RegexpFSALimitOrDefault() uint

RegexpFSALimitOrDefault returns the finite state automaton size limit or default.

type LRUSeriesCachePolicyConfiguration 

type LRUSeriesCachePolicyConfiguration struct {
	MaxBlocks         uint `yaml:"maxBlocks" validate:"nonzero"`
	EventsChannelSize uint `yaml:"eventsChannelSize" validate:"nonzero"`
}

LRUSeriesCachePolicyConfiguration contains configuration for the LRU series caching policy.

type LimitsConfiguration added in v0.15.3 

type LimitsConfiguration struct {
	// MaxRecentlyQueriedSeriesDiskBytesRead sets the upper limit on time series bytes
	// read from disk within a given lookback period. Queries which are issued while this
	// max is surpassed encounter an error.
	MaxRecentlyQueriedSeriesDiskBytesRead *MaxRecentQueryResourceLimitConfiguration `yaml:"maxRecentlyQueriedSeriesDiskBytesRead"`

	// MaxRecentlyQueriedSeriesDiskRead sets the upper limit on time series read from disk within a given lookback
	// period. Queries which are issued while this max is surpassed encounter an error.
	// This is the number of time series, which is different from the number of bytes controlled by
	// MaxRecentlyQueriedSeriesDiskBytesRead.
	MaxRecentlyQueriedSeriesDiskRead *MaxRecentQueryResourceLimitConfiguration `yaml:"maxRecentlyQueriedSeriesDiskRead"`

	// MaxRecentlyQueriedSeriesBlocks sets the upper limit on time series blocks
	// count within a given lookback period. Queries which are issued while this
	// max is surpassed encounter an error.
	MaxRecentlyQueriedSeriesBlocks *MaxRecentQueryResourceLimitConfiguration `yaml:"maxRecentlyQueriedSeriesBlocks"`

	// MaxOutstandingWriteRequests controls the maximum number of outstanding write requests
	// that the server will allow before it begins rejecting requests. Note that this value
	// is independent of the number of values that are being written (due to variable batch
	// size from the client) but is still very useful for enforcing backpressure due to the fact
	// that all writes within a single RPC are single-threaded.
	MaxOutstandingWriteRequests int `yaml:"maxOutstandingWriteRequests" validate:"min=0"`

	// MaxOutstandingReadRequests controls the maximum number of outstanding read requests that
	// the server will allow before it begins rejecting requests. Just like MaxOutstandingWriteRequests
	// this value is independent of the number of time series being read.
	MaxOutstandingReadRequests int `yaml:"maxOutstandingReadRequests" validate:"min=0"`

	// MaxOutstandingRepairedBytes controls the maximum number of bytes that can be loaded into memory
	// as part of the repair process. For example if the value was set to 2^31 then up to 2GiB of
	// repaired data could be "outstanding" in memory at one time. Once that limit was hit, the repair
	// process would pause until some of the repaired bytes had been persisted to disk (and subsequently
	// evicted from memory) at which point it would resume.
	MaxOutstandingRepairedBytes int64 `yaml:"maxOutstandingRepairedBytes" validate:"min=0"`

	// MaxEncodersPerBlock is the maximum number of encoders permitted in a block.
	// When there are too many encoders, merging them (during a tick) puts a high
	// load on the CPU, which can prevent other DB operations.
	// A setting of 0 means there is no maximum.
	MaxEncodersPerBlock int `yaml:"maxEncodersPerBlock" validate:"min=0"`

	// Write new series limit per second to limit overwhelming during new ID bursts.
	WriteNewSeriesPerSecond int `yaml:"writeNewSeriesPerSecond" validate:"min=0"`
}

LimitsConfiguration contains configuration for configurable limits that can be applied to M3DB.

type MaxCapacityPoolPolicy 

type MaxCapacityPoolPolicy struct {
	CapacityPoolPolicy `yaml:",inline"`

	// The max capacity of items in the pool.
	MaxCapacity *int `yaml:"maxCapacity"`
}

MaxCapacityPoolPolicy specifies a single pool policy that has a per element capacity, and a maximum allowed capacity as well.

func (*MaxCapacityPoolPolicy) MaxCapacityOrDefault added in v0.6.0 

func (p *MaxCapacityPoolPolicy) MaxCapacityOrDefault() int

MaxCapacityOrDefault returns the configured maximum capacity if present, or a default value otherwise.

type MaxRecentQueryResourceLimitConfiguration added in v0.15.15 

type MaxRecentQueryResourceLimitConfiguration struct {
	// Value sets the max value for the resource limit.
	Value int64 `yaml:"value" validate:"min=0"`
	// Lookback is the period in which a given resource limit is enforced.
	Lookback time.Duration `yaml:"lookback" validate:"min=0"`
}

MaxRecentQueryResourceLimitConfiguration sets an upper limit on resources consumed by all queries globally within a dbnode per some lookback period of time. Once exceeded, queries within that period of time will be abandoned.

type MmapConfiguration 

type MmapConfiguration struct {
	// HugeTLB is the huge pages configuration which will only take affect
	// on platforms that support it, currently just linux
	HugeTLB MmapHugeTLBConfiguration `yaml:"hugeTLB"`
}

MmapConfiguration is the mmap configuration.

func DefaultMmapConfiguration 

func DefaultMmapConfiguration() MmapConfiguration

DefaultMmapConfiguration is the default mmap configuration.

type MmapHugeTLBConfiguration 

type MmapHugeTLBConfiguration struct {
	// Enabled if true or disabled if false
	Enabled bool `yaml:"enabled"`

	// Threshold is the threshold on which to use the huge TLB flag if enabled
	Threshold int64 `yaml:"threshold"`
}

MmapHugeTLBConfiguration is the mmap huge TLB configuration.

type NamespaceProtoSchema added in v0.9.2 

type NamespaceProtoSchema struct {
	// For application m3db client integration test convenience (where a local dbnode is started as a docker container),
	// we allow loading user schema from local file into schema registry.
	SchemaFilePath string `yaml:"schemaFilePath"`
	MessageName    string `yaml:"messageName"`
}

NamespaceProtoSchema is the namespace protobuf schema.

func (NamespaceProtoSchema) Validate added in v0.9.2 

func (c NamespaceProtoSchema) Validate() error

Validate validates the NamespaceProtoSchema.

type PoolPolicy 

type PoolPolicy struct {
	// The size of the pool.
	Size *int `yaml:"size"`

	// The low watermark to start refilling the pool, if zero none.
	RefillLowWaterMark *float64 `yaml:"lowWatermark"`

	// The high watermark to stop refilling the pool, if zero none.
	RefillHighWaterMark *float64 `yaml:"highWatermark"`
}

PoolPolicy specifies a single pool policy.

func (*PoolPolicy) RefillHighWaterMarkOrDefault added in v0.6.0 

func (p *PoolPolicy) RefillHighWaterMarkOrDefault() float64

RefillHighWaterMarkOrDefault returns the configured refill high water mark if present, or a default value otherwise.

func (*PoolPolicy) RefillLowWaterMarkOrDefault added in v0.6.0 

func (p *PoolPolicy) RefillLowWaterMarkOrDefault() float64

RefillLowWaterMarkOrDefault returns the configured refill low water mark if present, or a default value otherwise.

func (*PoolPolicy) SizeOrDefault added in v0.6.0 

func (p *PoolPolicy) SizeOrDefault() int

SizeOrDefault returns the configured size if present, or a default value otherwise.

type PoolingPolicy 

type PoolingPolicy struct {
	// The initial alloc size for a block.
	BlockAllocSize *int `yaml:"blockAllocSize"`

	// The thrift bytes pool max bytes slice allocation for a single binary field.
	ThriftBytesPoolAllocSize *int `yaml:"thriftBytesPoolAllocSize"`

	// The general pool type (currently only supported: simple).
	Type *PoolingType `yaml:"type"`

	// The Bytes pool buckets to use.
	BytesPool BucketPoolPolicy `yaml:"bytesPool"`

	// The policy for the checked bytes wrapper pool.
	CheckedBytesWrapperPool PoolPolicy `yaml:"checkedBytesWrapperPool"`

	// The policy for the Closers pool.
	ClosersPool PoolPolicy `yaml:"closersPool"`

	// The policy for the Context pool.
	ContextPool PoolPolicy `yaml:"contextPool"`

	// The policy for the DatabaseSeries pool.
	SeriesPool PoolPolicy `yaml:"seriesPool"`

	// The policy for the DatabaseBlock pool.
	BlockPool PoolPolicy `yaml:"blockPool"`

	// The policy for the Encoder pool.
	EncoderPool PoolPolicy `yaml:"encoderPool"`

	// The policy for the Iterator pool.
	IteratorPool PoolPolicy `yaml:"iteratorPool"`

	// The policy for the Segment Reader pool.
	SegmentReaderPool PoolPolicy `yaml:"segmentReaderPool"`

	// The policy for the Identifier pool.
	IdentifierPool PoolPolicy `yaml:"identifierPool"`

	// The policy for the FetchBlockMetadataResult pool.
	FetchBlockMetadataResultsPool CapacityPoolPolicy `yaml:"fetchBlockMetadataResultsPool"`

	// The policy for the FetchBlocksMetadataResults pool.
	FetchBlocksMetadataResultsPool CapacityPoolPolicy `yaml:"fetchBlocksMetadataResultsPool"`

	// The policy for the ReplicaMetadataSlicePool pool.
	ReplicaMetadataSlicePool CapacityPoolPolicy `yaml:"replicaMetadataSlicePool"`

	// The policy for the BlockMetadat pool.
	BlockMetadataPool PoolPolicy `yaml:"blockMetadataPool"`

	// The policy for the BlockMetadataSlice pool.
	BlockMetadataSlicePool CapacityPoolPolicy `yaml:"blockMetadataSlicePool"`

	// The policy for the BlocksMetadata pool.
	BlocksMetadataPool PoolPolicy `yaml:"blocksMetadataPool"`

	// The policy for the BlocksMetadataSlice pool.
	BlocksMetadataSlicePool CapacityPoolPolicy `yaml:"blocksMetadataSlicePool"`

	// The policy for the tags pool.
	TagsPool MaxCapacityPoolPolicy `yaml:"tagsPool"`

	// The policy for the tags iterator pool.
	TagsIteratorPool PoolPolicy `yaml:"tagIteratorPool"`

	// The policy for the index.ResultsPool.
	IndexResultsPool PoolPolicy `yaml:"indexResultsPool"`

	// The policy for the TagEncoderPool.
	TagEncoderPool PoolPolicy `yaml:"tagEncoderPool"`

	// The policy for the TagDecoderPool.
	TagDecoderPool PoolPolicy `yaml:"tagDecoderPool"`

	// The policy for the WriteBatchPool.
	WriteBatchPool WriteBatchPoolPolicy `yaml:"writeBatchPool"`

	// The policy for the BufferBucket pool.
	BufferBucketPool PoolPolicy `yaml:"bufferBucketPool"`

	// The policy for the BufferBucketVersions pool.
	BufferBucketVersionsPool PoolPolicy `yaml:"bufferBucketVersionsPool"`

	// The policy for the RetrieveRequestPool pool.
	RetrieveRequestPool PoolPolicy `yaml:"retrieveRequestPool"`

	// The policy for the PostingsListPool.
	PostingsListPool PoolPolicy `yaml:"postingsListPool"`
}

PoolingPolicy specifies the pooling policy. To add a new pool, follow these steps:

  1. Add the pool to the struct below.
  2. Add the default values to the defaultPoolPolicies or defaultBucketPoolPolicies map.
  3. Add a call to initDefaultsAndValidate() for the new pool in the PoolingPolicy.InitDefaultsAndValidate() method.

func (*PoolingPolicy) BlockAllocSizeOrDefault added in v0.6.0 

func (p *PoolingPolicy) BlockAllocSizeOrDefault() int

BlockAllocSizeOrDefault returns the configured block alloc size if provided, or a default value otherwise.

func (*PoolingPolicy) InitDefaultsAndValidate added in v0.6.0 

func (p *PoolingPolicy) InitDefaultsAndValidate() error

InitDefaultsAndValidate initializes all default values and validates the configuration

func (*PoolingPolicy) ThriftBytesPoolAllocSizeOrDefault added in v0.11.0 

func (p *PoolingPolicy) ThriftBytesPoolAllocSizeOrDefault() int

ThriftBytesPoolAllocSizeOrDefault returns the configured thrift bytes pool max alloc size if provided, or a default value otherwise.

func (*PoolingPolicy) TypeOrDefault added in v0.6.0 

func (p *PoolingPolicy) TypeOrDefault() PoolingType

TypeOrDefault returns the configured pooling type if provided, or a default value otherwise.

type PoolingType 

type PoolingType string

PoolingType is a type of pooling, using runtime or mmap'd bytes pooling. 

const (
	// SimplePooling uses the basic Go runtime to allocate bytes for bytes pools.
	SimplePooling PoolingType = "simple"
)

type PostingsListCacheConfiguration added in v0.6.0 

type PostingsListCacheConfiguration struct {
	Size        *int  `yaml:"size"`
	CacheRegexp *bool `yaml:"cacheRegexp"`
	CacheTerms  *bool `yaml:"cacheTerms"`
}

PostingsListCacheConfiguration is the postings list cache configuration.

func (PostingsListCacheConfiguration) CacheRegexpOrDefault added in v0.6.0 

func (p PostingsListCacheConfiguration) CacheRegexpOrDefault() bool

CacheRegexpOrDefault returns the provided cache regexp configuration value or the default value is none is provided.

func (PostingsListCacheConfiguration) CacheTermsOrDefault added in v0.6.0 

func (p PostingsListCacheConfiguration) CacheTermsOrDefault() bool

CacheTermsOrDefault returns the provided cache terms configuration value or the default value is none is provided.

func (PostingsListCacheConfiguration) SizeOrDefault added in v0.6.0 

func (p PostingsListCacheConfiguration) SizeOrDefault() int

SizeOrDefault returns the provided size or the default value is none is provided.

type ProtoConfiguration added in v0.8.2 

type ProtoConfiguration struct {
	// Enabled specifies whether proto is enabled.
	Enabled        bool                            `yaml:"enabled"`
	SchemaRegistry map[string]NamespaceProtoSchema `yaml:"schema_registry"`
}

ProtoConfiguration is the configuration for running with ProtoDataMode enabled.

func (*ProtoConfiguration) Validate added in v0.8.2 

func (c *ProtoConfiguration) Validate() error

Validate validates the ProtoConfiguration.

type RegexpCacheConfiguration added in v1.0.0 

type RegexpCacheConfiguration struct {
	Size *int `yaml:"size"`
}

RegexpCacheConfiguration is a compiled regexp cache for query regexps.

func (RegexpCacheConfiguration) SizeOrDefault added in v1.0.0 

func (c RegexpCacheConfiguration) SizeOrDefault() int

SizeOrDefault returns the provided size or the default value is none is provided.

type RepairPolicy 

type RepairPolicy struct {
	// Enabled or disabled.
	Enabled bool `yaml:"enabled"`

	// The repair throttle.
	Throttle time.Duration `yaml:"throttle"`

	// The repair check interval.
	CheckInterval time.Duration `yaml:"checkInterval"`

	// Whether debug shadow comparisons are enabled.
	DebugShadowComparisonsEnabled bool `yaml:"debugShadowComparisonsEnabled"`

	// If enabled, what percentage of metadata should perform a detailed debug
	// shadow comparison.
	DebugShadowComparisonsPercentage float64 `yaml:"debugShadowComparisonsPercentage"`
}

RepairPolicy is the repair policy.

type ReplicatedCluster added in v0.11.0 

type ReplicatedCluster struct {
	Name          string                `yaml:"name"`
	RepairEnabled bool                  `yaml:"repairEnabled"`
	Client        *client.Configuration `yaml:"client"`
}

ReplicatedCluster defines a cluster to replicate data from.

func (*ReplicatedCluster) Validate added in v0.11.0 

func (r *ReplicatedCluster) Validate() error

Validate validates the configuration for a replicated cluster.

type ReplicationPolicy added in v0.11.0 

type ReplicationPolicy struct {
	Clusters []ReplicatedCluster `yaml:"clusters"`
}

ReplicationPolicy is the replication policy.

func (*ReplicationPolicy) Validate added in v0.11.0 

func (r *ReplicationPolicy) Validate() error

Validate validates the replication policy.

type SeriesCacheConfiguration 

type SeriesCacheConfiguration struct {
	Policy series.CachePolicy                 `yaml:"policy"`
	LRU    *LRUSeriesCachePolicyConfiguration `yaml:"lru"`
}

SeriesCacheConfiguration is the series cache configuration.

type TChannelConfiguration added in v0.15.0 

type TChannelConfiguration struct {
	// MaxIdleTime is the maximum idle time.
	MaxIdleTime time.Duration `yaml:"maxIdleTime"`
	// IdleCheckInterval is the idle check interval.
	IdleCheckInterval time.Duration `yaml:"idleCheckInterval"`
}

TChannelConfiguration holds TChannel config options.

type TickConfiguration 

type TickConfiguration struct {
	// Tick series batch size is the batch size to process series together
	// during a tick before yielding and sleeping the per series duration
	// multiplied by the batch size.
	// The higher this value is the more variable CPU utilization will be
	// but the shorter ticks will ultimately be.
	SeriesBatchSize int `yaml:"seriesBatchSize"`

	// Tick per series sleep at the completion of a tick batch.
	PerSeriesSleepDuration time.Duration `yaml:"perSeriesSleepDuration"`

	// Tick minimum interval controls the minimum tick interval for the node.
	MinimumInterval time.Duration `yaml:"minimumInterval"`
}

TickConfiguration is the tick configuration for background processing of series as blocks are rotated from mutable to immutable and out of order writes are merged.

type TransformConfiguration added in v0.9.0 

type TransformConfiguration struct {
	// TruncateBy determines what type of truncatation is applied to incoming
	// writes.
	TruncateBy series.TruncateType `yaml:"truncateBy"`
	// ForcedValue determines what to set all incoming write values to.
	ForcedValue *float64 `yaml:"forceValue"`
}

TransformConfiguration contains configuration options that can transform incoming writes.

func (*TransformConfiguration) Validate added in v0.9.0 

func (c *TransformConfiguration) Validate() error

Validate validates the transform configuration.

type WideConfiguration added in v1.0.0 

type WideConfiguration struct {
	// BatchSize represents batch size for wide operations. This size corresponds
	// to how many series are processed within a single "chunk"; larger batch
	// sizes will complete the query faster, but increase memory consumption.
	BatchSize int `yaml:"batchSize"`
}

WideConfiguration contains configuration for wide operations. These differ from regular paths by optimizing for query completeness across arbitary query ranges rather than speed.

type WriteBatchPoolPolicy added in v0.4.8 

type WriteBatchPoolPolicy struct {
	// The size of the pool.
	Size *int `yaml:"size"`

	// InitialBatchSize controls the initial batch size for each WriteBatch when
	// the pool is being constructed / refilled.
	InitialBatchSize *int `yaml:"initialBatchSize"`

	// MaxBatchSize controls the maximum size that a pooled WriteBatch can grow to
	// and still remain in the pool.
	MaxBatchSize *int `yaml:"maxBatchSize"`
}

WriteBatchPoolPolicy specifies the pooling policy for the WriteBatch pool.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.