config

package
v2.0.8-rc.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 25, 2022 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefaultSeedPeerConcurrentUploadLimit is default number for seed peer concurrent upload limit.
	DefaultSeedPeerConcurrentUploadLimit = 300

	// DefaultPeerConcurrentUploadLimit is default number for peer concurrent upload limit.
	DefaultPeerConcurrentUploadLimit = 50

	// DefaultPeerParallelCount is default number for pieces to download in parallel.
	DefaultPeerParallelCount = 4

	// DefaultSchedulerFilterParentLimit is default limit the number for filter traversals.
	DefaultSchedulerFilterParentLimit = 4

	// DefaultSchedulerFilterParentRangeLimit is default limit the range for filter traversals.
	DefaultSchedulerFilterParentRangeLimit = 40
)
View Source 
const (
	// DefaultSchedulerAlgorithm is default algorithm for scheduler.
	DefaultSchedulerAlgorithm = "default"

	// DefaultSchedulerBackSourceCount is default back-to-source count for scheduler.
	DefaultSchedulerBackSourceCount = 3

	// DefaultSchedulerRetryBackSourceLimit is default retry back-to-source limit for scheduler.
	DefaultSchedulerRetryBackSourceLimit = 5

	// DefaultSchedulerRetryLimit is default retry limit for scheduler.
	DefaultSchedulerRetryLimit = 10

	// DefaultSchedulerRetryInterval is default retry interval for scheduler.
	DefaultSchedulerRetryInterval = 50 * time.Millisecond

	// DefaultSchedulerPeerGCInterval is default interval for peer gc.
	DefaultSchedulerPeerGCInterval = 10 * time.Second

	// DefaultSchedulerPeerTTL is default ttl for peer.
	DefaultSchedulerPeerTTL = 24 * time.Hour

	// DefaultSchedulerTaskGCInterval is default interval for task gc.
	DefaultSchedulerTaskGCInterval = 30 * time.Minute

	// DefaultSchedulerHostGCInterval is default interval for host gc.
	DefaultSchedulerHostGCInterval = 1 * time.Hour

	// DefaultRefreshModelInterval is model refresh interval.
	DefaultRefreshModelInterval = 168 * time.Hour

	// DefaultCPU is default cpu usage.
	DefaultCPU = 1
)
View Source 
const (
	// DefaultManagerSchedulerClusterID is default id for scheduler cluster.
	DefaultManagerSchedulerClusterID = 1

	// DefaultManagerKeepAliveInterval is default interval for keepalive.
	DefaultManagerKeepAliveInterval = 5 * time.Second
)
View Source 
const (
	// DefaultJobGlobalWorkerNum is default global worker number for job.
	DefaultJobGlobalWorkerNum = 500

	// DefaultJobSchedulerWorkerNum is default scheduler worker number for job.
	DefaultJobSchedulerWorkerNum = 500

	// DefaultJobGlobalWorkerNum is default local worker number for job.
	DefaultJobLocalWorkerNum = 1000

	// DefaultJobRedisBrokerDB is default db for redis broker.
	DefaultJobRedisBrokerDB = 1

	// DefaultJobRedisBackendDB is default db for redis backend.
	DefaultJobRedisBackendDB = 2
)
View Source 
const (
	// DefaultStorageMaxSize is the default maximum size of record file.
	DefaultStorageMaxSize = 100

	// DefaultStorageMaxBackups is the default maximum count of backup.
	DefaultStorageMaxBackups = 10

	// DefaultStorageBufferSize is the default size of buffer container.
	DefaultStorageBufferSize = 100
)
View Source 
const (
	// DefaultDynConfigRefreshInterval is default refresh interval for dynamic configuration.
	DefaultDynConfigRefreshInterval = 10 * time.Second
)
View Source 
const (
	// DefaultMetricsAddr is default address for metrics server.
	DefaultMetricsAddr = ":8000"
)
View Source 
const (
	// DefaultServerPort is default port for server.
	DefaultServerPort = 8002
)

Variables

View Source 
var (
	// DefaultCertValidityPeriod is default validity period of certificate.
	DefaultCertValidityPeriod = 180 * 24 * time.Hour
)
View Source 
var (
	// DefaultNetworkEnableIPv6 is default value of enableIPv6.
	DefaultNetworkEnableIPv6 = false
)

Functions

This section is empty.

Types

type CertSpec added in v2.0.6 

type CertSpec struct {
	// ValidityPeriod is the validity period of certificate.
	ValidityPeriod time.Duration `mapstructure:"validityPeriod" yaml:"validityPeriod"`
}

type Config 

type Config struct {
	// Base options.
	base.Options `yaml:",inline" mapstructure:",squash"`

	// Scheduler configuration.
	Scheduler SchedulerConfig `yaml:"scheduler" mapstructure:"scheduler"`

	// Server configuration.
	Server ServerConfig `yaml:"server" mapstructure:"server"`

	// Dynconfig configuration.
	DynConfig DynConfig `yaml:"dynConfig" mapstructure:"dynConfig"`

	// Manager configuration.
	Manager ManagerConfig `yaml:"manager" mapstructure:"manager"`

	// SeedPeer configuration.
	SeedPeer SeedPeerConfig `yaml:"seedPeer" mapstructure:"seedPeer"`

	// Host configuration.
	Host HostConfig `yaml:"host" mapstructure:"host"`

	// Job configuration.
	Job JobConfig `yaml:"job" mapstructure:"job"`

	// Storage configuration.
	Storage StorageConfig `yaml:"storage" mapstructure:"storage"`

	// Metrics configuration.
	Metrics MetricsConfig `yaml:"metrics" mapstructure:"metrics"`

	// Security configuration.
	Security SecurityConfig `yaml:"security" mapstructure:"security"`

	// Network configuration.
	Network NetworkConfig `yaml:"network" mapstructure:"network"`
}

func New 

func New() *Config

New default configuration.

func (*Config) Convert 

func (cfg *Config) Convert() error

func (*Config) Validate 

func (cfg *Config) Validate() error

Validate config parameters.

type DynConfig 

type DynConfig struct {
	// RefreshInterval is refresh interval for manager cache.
	RefreshInterval time.Duration `yaml:"refreshInterval" mapstructure:"refreshInterval"`
}

type DynconfigData 

type DynconfigData struct {
	ID               uint64            `yaml:"id" mapstructure:"id" json:"id"`
	Hostname         string            `yaml:"hostname" mapstructure:"hostname" json:"host_name"`
	Idc              string            `yaml:"idc" mapstructure:"idc" json:"idc"`
	Location         string            `yaml:"location" mapstructure:"location" json:"location"`
	NetTopology      string            `yaml:"netTopology" mapstructure:"netTopology" json:"net_topology"`
	IP               string            `yaml:"ip" mapstructure:"ip" json:"ip"`
	Port             int32             `yaml:"port" mapstructure:"port" json:"port"`
	State            string            `yaml:"state" mapstructure:"state" json:"state"`
	SeedPeers        []*SeedPeer       `yaml:"seedPeers" mapstructure:"seedPeers" json:"seed_peers"`
	SchedulerCluster *SchedulerCluster `yaml:"schedulerCluster" mapstructure:"schedulerCluster" json:"scheduler_cluster"`
}

type DynconfigInterface 

type DynconfigInterface interface {
	// Get the dynamic schedulers resolve addrs.
	GetResolveSeedPeerAddrs() ([]resolver.Address, error)

	// Get the dynamic seed peers config from manager.
	GetSeedPeers() ([]*SeedPeer, error)

	// Get the scheduler cluster config.
	GetSchedulerClusterConfig() (types.SchedulerClusterConfig, bool)

	// Get the client config.
	GetSchedulerClusterClientConfig() (types.SchedulerClusterClientConfig, bool)

	// Get the dynamic config from manager.
	Get() (*DynconfigData, error)

	// Refresh refreshes dynconfig in cache.
	Refresh() error

	// Register allows an instance to register itself to listen/observe events.
	Register(Observer)

	// Deregister allows an instance to remove itself from the collection of observers/listeners.
	Deregister(Observer)

	// Notify publishes new events to listeners.
	Notify() error

	// Serve the dynconfig listening service.
	Serve() error

	// Stop the dynconfig listening service.
	Stop() error
}

func NewDynconfig 

func NewDynconfig(rawManagerClient managerclient.Client, cacheDir string, cfg *Config) (DynconfigInterface, error)

NewDynconfig returns a new dynconfig instence.

type GCConfig 

type GCConfig struct {
	// Peer gc interval.
	PeerGCInterval time.Duration `yaml:"peerGCInterval" mapstructure:"peerGCInterval"`

	// Peer time to live.
	PeerTTL time.Duration `yaml:"peerTTL" mapstructure:"peerTTL"`

	// Task gc interval.
	TaskGCInterval time.Duration `yaml:"taskGCInterval" mapstructure:"taskGCInterval"`

	// Host gc interval.
	HostGCInterval time.Duration `yaml:"hostGCInterval" mapstructure:"hostGCInterval"`
}

type HostConfig 

type HostConfig struct {
	// IDC for scheduler.
	IDC string `mapstructure:"idc" yaml:"idc"`

	// NetTopology for scheduler.
	NetTopology string `mapstructure:"netTopology" yaml:"netTopology"`

	// Location for scheduler.
	Location string `mapstructure:"location" yaml:"location"`
}

type JobConfig 

type JobConfig struct {
	// Enable job service.
	Enable bool `yaml:"enable" mapstructure:"enable"`

	// Number of workers in global queue.
	GlobalWorkerNum uint `yaml:"globalWorkerNum" mapstructure:"globalWorkerNum"`

	// Number of workers in scheduler queue.
	SchedulerWorkerNum uint `yaml:"schedulerWorkerNum" mapstructure:"schedulerWorkerNum"`

	// Number of workers in local queue.
	LocalWorkerNum uint `yaml:"localWorkerNum" mapstructure:"localWorkerNum"`

	// Redis configuration.
	Redis RedisConfig `yaml:"redis" mapstructure:"redis"`
}

type KeepAliveConfig 

type KeepAliveConfig struct {
	// Keep alive interval.
	Interval time.Duration `yaml:"interval" mapstructure:"interval"`
}

type ManagerConfig 

type ManagerConfig struct {
	// Addr is manager address.
	Addr string `yaml:"addr" mapstructure:"addr"`

	// SchedulerClusterID is scheduler cluster id.
	SchedulerClusterID uint `yaml:"schedulerClusterID" mapstructure:"schedulerClusterID"`

	// KeepAlive configuration.
	KeepAlive KeepAliveConfig `yaml:"keepAlive" mapstructure:"keepAlive"`
}

type MetricsConfig added in v2.0.1 

type MetricsConfig struct {
	// Enable metrics service.
	Enable bool `yaml:"enable" mapstructure:"enable"`

	// Metrics service address.
	Addr string `yaml:"addr" mapstructure:"addr"`

	// Enable peer host metrics.
	EnablePeerHost bool `yaml:"enablePeerHost" mapstructure:"enablePeerHost"`
}

type NetworkConfig added in v2.0.7 

type NetworkConfig struct {
	// EnableIPv6 enables ipv6 for server.
	EnableIPv6 bool `mapstructure:"enableIPv6" yaml:"enableIPv6"`
}

type Observer 

type Observer interface {
	// OnNotify allows an event to be published to interface implementations.
	OnNotify(*DynconfigData)
}

type RedisConfig 

type RedisConfig struct {
	// DEPRECATED: Please use the `addrs` field instead.
	Host string `yaml:"host" mapstructure:"host"`

	// DEPRECATED: Please use the `addrs` field instead.
	Port int `yaml:"port" mapstructure:"port"`

	// Server addresses.
	Addrs []string `yaml:"addrs" mapstructure:"addrs"`

	// Server username.
	Username string `yaml:"username" mapstructure:"username"`

	// Server password.
	Password string `yaml:"password" mapstructure:"password"`

	// Broker database name.
	BrokerDB int `yaml:"brokerDB" mapstructure:"brokerDB"`

	// Backend database name.
	BackendDB int `yaml:"backendDB" mapstructure:"backendDB"`
}

type SchedulerCluster added in v2.0.1 

type SchedulerCluster struct {
	ID           uint64 `yaml:"id" mapstructure:"id" json:"id"`
	Name         string `yaml:"name" mapstructure:"name" json:"name"`
	Config       []byte `yaml:"config" mapstructure:"config" json:"config"`
	ClientConfig []byte `yaml:"clientConfig" mapstructure:"clientConfig" json:"client_config"`
}

type SchedulerConfig 

type SchedulerConfig struct {
	// Scheduling algorithm used by the scheduler.
	Algorithm string `yaml:"algorithm" mapstructure:"algorithm"`

	// Single task allows the client to back-to-source count.
	BackSourceCount int `yaml:"backSourceCount" mapstructure:"backSourceCount"`

	// Retry scheduling back-to-source limit times.
	RetryBackSourceLimit int `yaml:"retryBackSourceLimit" mapstructure:"retryBackSourceLimit"`

	// Retry scheduling limit times.
	RetryLimit int `yaml:"retryLimit" mapstructure:"retryLimit"`

	// Retry scheduling interval.
	RetryInterval time.Duration `yaml:"retryInterval" mapstructure:"retryInterval"`

	// Task and peer gc configuration.
	GC GCConfig `yaml:"gc" mapstructure:"gc"`

	// Training configuration.
	Training TrainingConfig `yaml:"training" mapstructure:"training"`
}

type SecurityConfig added in v2.0.6 

type SecurityConfig struct {
	// AutoIssueCert indicates to issue client certificates for all grpc call
	// if AutoIssueCert is false, any other option in Security will be ignored.
	AutoIssueCert bool `mapstructure:"autoIssueCert" yaml:"autoIssueCert"`

	// CACert is the root CA certificate for all grpc tls handshake, it can be path or PEM format string.
	CACert types.PEMContent `mapstructure:"caCert" yaml:"caCert"`

	// TLSVerify indicates to verify client certificates.
	TLSVerify bool `mapstructure:"tlsVerify" yaml:"tlsVerify"`

	// TLSPolicy controls the grpc shandshake behaviors:
	// force: both ClientHandshake and ServerHandshake are only support tls.
	// prefer: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support tls.
	// default: ServerHandshake supports tls and insecure (non-tls), ClientHandshake will only support insecure (non-tls).
	TLSPolicy string `mapstructure:"tlsPolicy" yaml:"tlsPolicy"`

	// CertSpec is the desired state of certificate.
	CertSpec CertSpec `mapstructure:"certSpec" yaml:"certSpec"`
}

type SeedPeer added in v2.0.3 

type SeedPeer struct {
	ID              uint             `yaml:"id" mapstructure:"id" json:"id"`
	Hostname        string           `yaml:"hostname" mapstructure:"hostname" json:"host_name"`
	Type            string           `yaml:"type" mapstructure:"type" json:"type"`
	IDC             string           `yaml:"idc" mapstructure:"idc" json:"idc"`
	NetTopology     string           `yaml:"netTopology" mapstructure:"netTopology" json:"net_topology"`
	Location        string           `yaml:"location" mapstructure:"location" json:"location"`
	IP              string           `yaml:"ip" mapstructure:"ip" json:"ip"`
	Port            int32            `yaml:"port" mapstructure:"port" json:"port"`
	DownloadPort    int32            `yaml:"downloadPort" mapstructure:"downloadPort" json:"download_port"`
	SeedPeerCluster *SeedPeerCluster `yaml:"seedPeerCluster" mapstructure:"seedPeerCluster" json:"seed_peer_cluster"`
}

func (*SeedPeer) GetSeedPeerClusterConfig added in v2.0.3 

func (c *SeedPeer) GetSeedPeerClusterConfig() (types.SeedPeerClusterConfig, bool)

type SeedPeerCluster added in v2.0.3 

type SeedPeerCluster struct {
	ID     uint64 `yaml:"id" mapstructure:"id" json:"id"`
	Name   string `yaml:"name" mapstructure:"name" json:"name"`
	Config []byte `yaml:"config" mapstructure:"config" json:"config"`
}

type SeedPeerConfig added in v2.0.3 

type SeedPeerConfig struct {
	// Enable is to enable seed peer as P2P peer.
	Enable bool `yaml:"enable" mapstructure:"enable"`
}

type ServerConfig 

type ServerConfig struct {
	// DEPRECATED: Please use the `advertiseIP` field instead.
	IP string `yaml:"ip" mapstructure:"ip"`

	// DEPRECATED: Please use the `listenIP` field instead.
	Listen string `yaml:"listen" mapstructure:"listen"`

	// AdvertiseIP is advertise ip.
	AdvertiseIP string `yaml:"advertiseIP" mapstructure:"advertiseIP"`

	// ListenIP is listen ip, like: 0.0.0.0, 192.168.0.1.
	ListenIP string `yaml:"listenIP" mapstructure:"listenIP"`

	// Server port.
	Port int `yaml:"port" mapstructure:"port"`

	// Server hostname.
	Host string `yaml:"host" mapstructure:"host"`

	// Server work directory.
	WorkHome string `yaml:"workHome" mapstructure:"workHome"`

	// Server dynamic config cache directory.
	CacheDir string `yaml:"cacheDir" mapstructure:"cacheDir"`

	// Server log directory.
	LogDir string `yaml:"logDir" mapstructure:"logDir"`

	// Server plugin directory.
	PluginDir string `yaml:"pluginDir" mapstructure:"pluginDir"`

	// Server storage data directory.
	DataDir string `yaml:"dataDir" mapstructure:"dataDir"`
}

type StorageConfig added in v2.0.3 

type StorageConfig struct {
	// MaxSize sets the maximum size in megabytes of storage file.
	MaxSize int `yaml:"maxSize" mapstructure:"maxSize"`

	// MaxBackups sets the maximum number of storage files to retain.
	MaxBackups int `yaml:"maxBackups" mapstructure:"maxBackups"`

	// BufferSize sets the size of buffer container,
	// if the buffer is full, write all the records in the buffer to the file.
	BufferSize int `yaml:"bufferSize" mapstructure:"bufferSize"`
}

type TrainingConfig added in v2.0.5 

type TrainingConfig struct {
	// Enable training.
	Enable bool `yaml:"enable" mapstructure:"enable"`

	// Enable auto refresh model.
	EnableAutoRefresh bool `yaml:"enableAutoRefresh" mapstructure:"enableAutoRefresh"`

	// RefreshModelInterval is refresh interval for refreshing model.
	RefreshModelInterval time.Duration `yaml:"refreshModelInterval" mapstructure:"refreshModelInterval"`

	// CPU limit while training.
	CPU int `yaml:"cpu" mapstructure:"cpu"`
}

Source Files

View all Source files

Directories

Path Synopsis
Package mocks is a generated GoMock package.
 Package mocks is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.