config

package
v0.1.0-beta-3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 12, 2021 License: Apache-2.0 Imports: 30 Imported by: 0

Documentation

Overview

Package config holds all DaemonConfig of dfget.

Package config holds all options of peerhost.

Index

Constants

View Source 
const (
	BackSourceReasonNone          = 0
	BackSourceReasonRegisterFail  = 1
	BackSourceReasonMd5NotMatch   = 2
	BackSourceReasonDownloadError = 3
	BackSourceReasonNoSpace       = 4
	BackSourceReasonInitError     = 5
	BackSourceReasonWriteError    = 6
	BackSourceReasonHostSysError  = 7
	BackSourceReasonNodeEmpty     = 8
	BackSourceReasonSourceError   = 10
	BackSourceReasonUserSpecified = 100
	ForceNotBackSourceAddition    = 1000
)

the reason of backing to source

View Source 
const (
	PatternP2P    = "p2p"
	PatternCDN    = "cdn"
	PatternSource = "source"
)

download pattern

View Source 
const (
	DefaultPerPeerDownloadLimit = 20 * unit.MB
	DefaultTotalDownloadLimit   = 100 * unit.MB
	DefaultUploadLimit          = 100 * unit.MB
	DefaultMinRate              = 64 * unit.KB
)
View Source 
const (
	DefaultTimestampFormat = "2006-01-02 15:04:05"
	SchemaHTTP             = "http"

	ServerPortLowerLimit = 15000
	ServerPortUpperLimit = 65000

	DefaultTaskExpireTime  = 3 * time.Minute
	DefaultGCInterval      = 1 * time.Minute
	DefaultDaemonAliveTime = 5 * time.Minute
	DefaultScheduleTimeout = 5 * time.Minute
	DefaultDownloadTimeout = 5 * time.Minute

	DefaultSupernodeSchema = "http"
	DefaultSupernodeIP     = "127.0.0.1"
	DefaultSupernodePort   = 8002
)

others

View Source 
const (
	SimpleLocalTaskStoreStrategy  = StoreStrategy("io.d7y.storage.v2.simple")
	AdvanceLocalTaskStoreStrategy = StoreStrategy("io.d7y.storage.v2.advance")
)
View Source 
const (
	AttributePeerHost          = attribute.Key("d7y.peer.host")
	AttributeTaskID            = attribute.Key("d7y.peer.task.id")
	AttributeTaskContentLength = attribute.Key("d7y.peer.task.content_length")
	AttributePeerID            = attribute.Key("d7y.peer.id")
	AttributeTargetPeerID      = attribute.Key("d7y.peer.target.id")
	AttributeReusePeerID       = attribute.Key("d7y.peer.reuse.id")
	AttributeTargetPeerAddr    = attribute.Key("d7y.peer.target.addr")
	AttributeMainPeer          = attribute.Key("d7y.peer.task.main_peer")
	AttributePeerPacketCode    = attribute.Key("d7y.peer.packet.code")
	AttributePeerTaskSizeScope = attribute.Key("d7y.peer.size.scope")
	AttributePeerTaskSuccess   = attribute.Key("d7y.peer.task.success")
	AttributePeerTaskCode      = attribute.Key("d7y.peer.task.code")
	AttributePeerTaskMessage   = attribute.Key("d7y.peer.task.message")
	AttributePiece             = attribute.Key("d7y.peer.piece")
	AttributePieceWorker       = attribute.Key("d7y.peer.piece.worker")
	AttributePieceSuccess      = attribute.Key("d7y.peer.piece.success")
	AttributeGetPieceStartNum  = attribute.Key("d7y.peer.piece.start")
	AttributeGetPieceLimit     = attribute.Key("d7y.peer.piece.limit")
	AttributeGetPieceCount     = attribute.Key("d7y.peer.piece.count")
	AttributeGetPieceRetry     = attribute.Key("d7y.peer.piece.retry")
	AttributeWritePieceSuccess = attribute.Key("d7y.peer.piece.write.success")

	SpanFilePeerTask    = "file-peer-task"
	SpanStreamPeerTask  = "stream-peer-task"
	SpanReusePeerTask   = "reuse-peer-task"
	SpanRegisterTask    = "register"
	SpanFirstSchedule   = "schedule-#1"
	SpanGetPieceTasks   = "get-piece-tasks"
	SpanDownloadPiece   = "download-piece-#%d"
	SpanProxy           = "proxy"
	SpanWritePiece      = "write-piece"
	SpanWriteBackPiece  = "write-back-piece"
	SpanWaitPieceLimit  = "wait-limit"
	SpanPushPieceResult = "push-result"
	SpanPeerGC          = "peer-gc"
)
View Source 
const (
	HeaderDragonflyFilter = "X-Dragonfly-Filter"
	HeaderDragonflyPeer   = "X-Dragonfly-Peer"
	HeaderDragonflyTask   = "X-Dragonfly-Task"
	HeaderDragonflyBiz    = "X-Dragonfly-Biz"
)

Variables

View Source 
var DefaultSupernodesValue = &SupernodesValue{
	Nodes: []string{
		fmt.Sprintf("%s:%d", DefaultSupernodeIP, DefaultSupernodePort),
	},
}
View Source 
var (
	PeerHostConfigPath = "/etc/dragonfly/dfget-daemon.yaml"
)

Functions

func MkdirAll 

func MkdirAll(dir string, perm os.FileMode, uid, gid int) error

MkdirAll make directories recursive, and changes uid, gid to latest directory. For example: the path /data/x exists, uid=1, gid=1 when call MkdirAll("/data/x/y/z", 0755, 2, 2) MkdirAll creates /data/x/y and change owner to 2:2, creates /data/x/y/z and change owner to 2:2

Types

type BasicAuth 

type BasicAuth struct {
	Username string `json:"username" yaml:"username"`
	Password string `json:"password" yaml:"password"`
}

type CertPool 

type CertPool struct {
	Files []string
	*x509.CertPool
}

CertPool is a wrapper around x509.CertPool, which can be unmarshalled and constructed from a list of filenames.

func (*CertPool) MarshalJSON 

func (cp *CertPool) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaller to print the cert pool.

func (*CertPool) MarshalYAML 

func (cp *CertPool) MarshalYAML() (interface{}, error)

MarshalYAML implements yaml.Marshaller to print the cert pool.

func (*CertPool) UnmarshalJSON 

func (cp *CertPool) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler.

func (*CertPool) UnmarshalYAML 

func (cp *CertPool) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaler.

type ClientOption 

type ClientOption struct {
	base.Options `yaml:",inline" mapstructure:",squash"`
	// URL download URL.
	URL string `yaml:"url,omitempty" mapstructure:"url,omitempty"`

	// Output full output path.
	Output string `yaml:"output,omitempty" mapstructure:"output,omitempty"`

	// Timeout download timeout(second).
	Timeout time.Duration `yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"`

	BenchmarkRate unit.Bytes `yaml:"benchmarkRate,omitempty" mapstructure:"benchmarkRate,omitempty"`

	// Md5 expected file md5.
	// Deprecated: Md5 is deprecated, use DigestMethod with DigestValue instead
	Md5    string `yaml:"md5,omitempty" mapstructure:"md5,omitempty"`
	Digest string `yaml:"digest,omitempty" mapstructure:"digest,omitempty"`
	// DigestMethod indicates digest method, like md5, sha256
	DigestMethod string `yaml:"digestMethod,omitempty" mapstructure:"digestMethod,omitempty"`

	// DigestValue indicates digest value
	DigestValue string `yaml:"digestValue,omitempty" mapstructure:"digestValue,omitempty"`

	// Tag identify download task, it is available merely when md5 param not exist.
	Tag string `yaml:"tag,omitempty" mapstructure:"tag,omitempty"`

	// CallSystem system name that executes dfget.
	CallSystem string `yaml:"callSystem,omitempty" mapstructure:"callSystem,omitempty"`

	// Pattern download pattern, must be 'p2p' or 'cdn' or 'source',
	// default:`p2p`.
	Pattern string `yaml:"pattern,omitempty" mapstructure:"pattern,omitempty"`

	// CA certificate to verify when supernode interact with the source.
	Cacerts []string `yaml:"cacert,omitempty" mapstructure:"cacert,omitempty"`

	// Filter filter some query params of url, use char '&' to separate different params.
	// eg: -f 'key&sign' will filter 'key' and 'sign' query param.
	// in this way, different urls correspond one same download task that can use p2p mode.
	Filter string `yaml:"filter,omitempty" mapstructure:"filter,omitempty"`

	// Header of http request.
	// eg: --header='Accept: *' --header='Host: abc'.
	Header []string `yaml:"header,omitempty" mapstructure:"header,omitempty"`

	// DisableBackSource indicates whether to not back source to download when p2p fails.
	DisableBackSource bool `yaml:"disableBackSource,omitempty" mapstructure:"disableBackSource,omitempty"`

	// Insecure indicates whether skip secure verify when supernode interact with the source.
	Insecure bool `yaml:"insecure,omitempty" mapstructure:"insecure,omitempty"`

	// ShowProgress shows progress bar, it's conflict with `--console`.
	ShowProgress bool `yaml:"show-progress,omitempty" mapstructure:"show-progress,omitempty"`

	RateLimit rate.Limit `yaml:"rateLimit,omitempty" mapstructure:"rateLimit,omitempty"`

	// MoreDaemonOptions indicates more options passed to daemon by command line.
	MoreDaemonOptions string `yaml:"moreDaemonOptions,omitempty" mapstructure:"moreDaemonOptions,omitempty"`
}

ClientOption holds all the runtime config information.

func NewDfgetConfig 

func NewDfgetConfig() *ClientOption

func (*ClientOption) Convert 

func (cfg *ClientOption) Convert(args []string) error

func (*ClientOption) String 

func (cfg *ClientOption) String() string

func (*ClientOption) Validate 

func (cfg *ClientOption) Validate() error

type DaemonConfig 

type DaemonConfig = DaemonOption

type DaemonOption 

type DaemonOption struct {
	base.Options `yaml:",inline" mapstructure:",squash"`
	// AliveTime indicates alive duration for which daemon keeps no accessing by any uploading and download requests,
	// after this period daemon will automatically exit
	// when AliveTime == 0, will run infinitely
	AliveTime  clientutil.Duration `mapstructure:"aliveTime" yaml:"aliveTime"`
	GCInterval clientutil.Duration `mapstructure:"gcInterval" yaml:"gcInterval"`

	DataDir     string `mapstructure:"dataDir" yaml:"dataDir"`
	WorkHome    string `mapstructure:"workHome" yaml:"workHome"`
	KeepStorage bool   `mapstructure:"keepStorage" yaml:"keepStorage"`

	Scheduler    SchedulerOption `mapstructure:"scheduler" yaml:"scheduler"`
	Host         HostOption      `mapstructure:"host" yaml:"host"`
	Download     DownloadOption  `mapstructure:"download" yaml:"download"`
	Proxy        *ProxyOption    `mapstructure:"proxy" yaml:"proxy"`
	Upload       UploadOption    `mapstructure:"upload" yaml:"upload"`
	Storage      StorageOption   `mapstructure:"storage" yaml:"storage"`
	ConfigServer string          `mapstructure:"configServer" yaml:"configServer"`
}

func NewDaemonConfig 

func NewDaemonConfig() *DaemonOption

func (*DaemonOption) Convert 

func (p *DaemonOption) Convert() error

func (*DaemonOption) Load 

func (p *DaemonOption) Load(path string) error

func (*DaemonOption) Validate 

func (p *DaemonOption) Validate() error

type DfgetConfig 

type DfgetConfig = ClientOption

type DownloadOption 

type DownloadOption struct {
	TotalRateLimit   clientutil.RateLimit `mapstructure:"totalRateLimit" yaml:"totalRateLimit"`
	PerPeerRateLimit clientutil.RateLimit `mapstructure:"perPeerRateLimit" yaml:"perPeerRateLimit"`
	DownloadGRPC     ListenOption         `mapstructure:"downloadGRPC" yaml:"downloadGRPC"`
	PeerGRPC         ListenOption         `mapstructure:"peerGRPC" yaml:"peerGRPC"`
	CalculateDigest  bool                 `mapstructure:"calculateDigest" yaml:"calculateDigest"`
}

type DurationValue 

type DurationValue time.Duration

DurationValue supports time.Duration format like 30s, 1m30s, 1h and also treat integer as seconds

func NewDurationValue 

func NewDurationValue(p *time.Duration) *DurationValue

func (*DurationValue) Set 

func (d *DurationValue) Set(s string) error

func (*DurationValue) String 

func (d *DurationValue) String() string

func (*DurationValue) Type 

func (d *DurationValue) Type() string

type FileString 

type FileString string

func (*FileString) UnmarshalJSON 

func (f *FileString) UnmarshalJSON(b []byte) error

func (*FileString) UnmarshalYAML 

func (f *FileString) UnmarshalYAML(node *yaml.Node) error

type HijackConfig 

type HijackConfig struct {
	Cert  string        `yaml:"cert" mapstructure:"cert"`
	Key   string        `yaml:"key" mapstructure:"key"`
	Hosts []*HijackHost `yaml:"hosts" mapstructure:"hosts"`
}

HijackConfig represents how dfdaemon hijacks http requests.

type HijackHost 

type HijackHost struct {
	Regx     *Regexp   `yaml:"regx" mapstructure:"regx"`
	Insecure bool      `yaml:"insecure" mapstructure:"insecure"`
	Certs    *CertPool `yaml:"certs" mapstructure:"certs"`
}

HijackHost is a hijack rule for the hosts that matches Regx.

type HostOption 

type HostOption struct {
	// SecurityDomain is the security domain
	SecurityDomain string `mapstructure:"securityDomain" yaml:"securityDomain"`
	// Peerhost location for scheduler
	Location string `mapstructure:"location" yaml:"location"`
	// Peerhost idc for scheduler
	IDC string `mapstructure:"idc" yaml:"idc"`
	// Peerhost net topology for scheduler
	NetTopology string `mapstructure:"netTopology" yaml:"netTopology"`
	// The listen ip for all tcp services of daemon
	ListenIP string `mapstructure:"listenIP" yaml:"listenIP"`
	// The ip report to scheduler, normal same with listen ip
	AdvertiseIP string `mapstructure:"advertiseIP" yaml:"advertiseIP"`
}

type ListenOption 

type ListenOption struct {
	Security   SecurityOption    `mapstructure:"security" yaml:"security"`
	TCPListen  *TCPListenOption  `mapstructure:"tcpListen,omitempty" yaml:"tcpListen,omitempty"`
	UnixListen *UnixListenOption `mapstructure:"unixListen,omitempty" yaml:"unixListen,omitempty"`
}

type NetAddrsValue 

type NetAddrsValue struct {
	// contains filtered or unexported fields
}

func NewNetAddrsValue 

func NewNetAddrsValue(n *[]dfnet.NetAddr) *NetAddrsValue

func (*NetAddrsValue) Set 

func (nv *NetAddrsValue) Set(value string) error

func (*NetAddrsValue) String 

func (nv *NetAddrsValue) String() string

func (*NetAddrsValue) Type 

func (nv *NetAddrsValue) Type() string

type Proxy 

type Proxy struct {
	Regx     *Regexp `yaml:"regx" mapstructure:"regx"`
	UseHTTPS bool    `yaml:"useHTTPS" mapstructure:"useHTTPS"`
	Direct   bool    `yaml:"direct" mapstructure:"direct"`

	// Redirect is the host to redirect to, if not empty
	Redirect string `yaml:"redirect" mapstructure:"redirect"`
}

Proxy describes a regular expression matching rule for how to proxy a request.

func NewProxy 

func NewProxy(regx string, useHTTPS bool, direct bool, redirect string) (*Proxy, error)

func (*Proxy) Match 

func (r *Proxy) Match(url string) bool

Match checks if the given url matches the rule.

type ProxyOption 

type ProxyOption struct {
	// WARNING: when add more option, please update ProxyOption.unmarshal function
	ListenOption   `mapstructure:",squash" yaml:",inline"`
	BasicAuth      *BasicAuth      `mapstructure:"basicAuth" yaml:"basicAuth"`
	DefaultFilter  string          `mapstructure:"defaultFilter" yaml:"defaultFilter"`
	MaxConcurrency int64           `mapstructure:"maxConcurrency" yaml:"maxConcurrency"`
	RegistryMirror *RegistryMirror `mapstructure:"registryMirror" yaml:"registryMirror"`
	WhiteList      []*WhiteList    `mapstructure:"whiteList" yaml:"whiteList"`
	Proxies        []*Proxy        `mapstructure:"proxies" yaml:"proxies"`
	HijackHTTPS    *HijackConfig   `mapstructure:"hijackHTTPS" yaml:"hijackHTTPS"`
}

func (*ProxyOption) UnmarshalJSON 

func (p *ProxyOption) UnmarshalJSON(b []byte) error

func (*ProxyOption) UnmarshalYAML 

func (p *ProxyOption) UnmarshalYAML(node *yaml.Node) error

type RateLimitValue 

type RateLimitValue struct {
	// contains filtered or unexported fields
}

func NewLimitRateValue 

func NewLimitRateValue(rate *clientutil.RateLimit) *RateLimitValue

func (*RateLimitValue) Set 

func (r *RateLimitValue) Set(s string) error

func (*RateLimitValue) String 

func (r *RateLimitValue) String() string

func (*RateLimitValue) Type 

func (r *RateLimitValue) Type() string

type Regexp 

type Regexp struct {
	*regexp.Regexp
}

Regexp is a simple wrapper around regexp. Regexp to make it unmarshallable from a string.

func NewRegexp 

func NewRegexp(exp string) (*Regexp, error)

NewRegexp returns a new Regexp instance compiled from the given string.

func (*Regexp) MarshalJSON 

func (r *Regexp) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaller to print the regexp.

func (*Regexp) MarshalYAML 

func (r *Regexp) MarshalYAML() (interface{}, error)

MarshalYAML implements yaml.Marshaller to print the regexp.

func (*Regexp) UnmarshalJSON 

func (r *Regexp) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler.

func (*Regexp) UnmarshalYAML 

func (r *Regexp) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaler.

type RegistryMirror 

type RegistryMirror struct {
	// Remote url for the registry mirror, default is https://index.docker.io
	Remote *URL `yaml:"url" mapstructure:"url"`

	// Optional certificates if the mirror uses self-signed certificates
	Certs *CertPool `yaml:"certs" mapstructure:"certs"`

	// Whether to ignore certificates errors for the registry
	Insecure bool `yaml:"insecure" mapstructure:"insecure"`

	// Request the remote registry directly.
	Direct bool `yaml:"direct" mapstructure:"direct"`
}

RegistryMirror configures the mirror of the official docker registry

func (*RegistryMirror) TLSConfig 

func (r *RegistryMirror) TLSConfig() *tls.Config

TLSConfig returns the tls.Config used to communicate with the mirror.

type SchedulerOption 

type SchedulerOption struct {
	// NetAddrs is scheduler addresses.
	NetAddrs []dfnet.NetAddr `mapstructure:"netAddrs" yaml:"netAddrs"`

	// ScheduleTimeout is request timeout.
	ScheduleTimeout clientutil.Duration `mapstructure:"scheduleTimeout" yaml:"scheduleTimeout"`
}

type SecurityOption 

type SecurityOption struct {
	// Insecure indicate enable tls or not
	Insecure  bool        `mapstructure:"insecure" yaml:"insecure"`
	CACert    string      `mapstructure:"caCert" yaml:"caCert"`
	Cert      string      `mapstructure:"cert" yaml:"cert"`
	Key       string      `mapstructure:"key" yaml:"key"`
	TLSConfig *tls.Config `mapstructure:"tlsConfig" yaml:"tlsConfig"`
}

type StorageOption 

type StorageOption struct {
	// DataPath indicates directory which stores temporary files for p2p uploading
	DataPath string `mapstructure:"dataPath" yaml:"dataPath"`
	// TaskExpireTime indicates caching duration for which cached file keeps no accessed by any process,
	// after this period cache file will be gc
	TaskExpireTime clientutil.Duration `mapstructure:"taskExpireTime" yaml:"taskExpireTime"`
	// DiskGCThreshold indicates the threshold to gc the oldest tasks
	DiskGCThreshold unit.Bytes `mapstructure:"diskGCThreshold" yaml:"diskGCThreshold"`
	// Multiplex indicates reusing underlying storage for same task id
	Multiplex     bool          `mapstructure:"multiplex" yaml:"multiplex"`
	StoreStrategy StoreStrategy `mapstructure:"strategy" yaml:"strategy"`
}

type StoreStrategy 

type StoreStrategy string

type SupernodesValue 

type SupernodesValue struct {
	Nodes []string
}

func (*SupernodesValue) Set 

func (sv *SupernodesValue) Set(value string) error

Set implements the pflag.Value interface.

func (*SupernodesValue) String 

func (sv *SupernodesValue) String() string

String implements the pflag.Value interface.

func (*SupernodesValue) Type 

func (sv *SupernodesValue) Type() string

Type implements the pflag.Value interface.

type TCPListenOption 

type TCPListenOption struct {
	// Listen stands listen interface, like: 0.0.0.0, 192.168.0.1
	Listen string `mapstructure:"listen" yaml:"listen"`

	// PortRange stands listen port
	// yaml example 1:
	//   port: 12345
	// yaml example 2:
	//   port:
	//     start: 12345
	//     end: 12346
	PortRange TCPListenPortRange `mapstructure:"port" yaml:"port"`

	// Namespace stands the linux net namespace, like /proc/1/ns/net
	// It's useful for running daemon in pod with ip allocated and listen in host
	Namespace string `mapstructure:"namespace" yaml:"namespace"`
}

type TCPListenPortRange 

type TCPListenPortRange struct {
	Start int
	End   int
}

func (*TCPListenPortRange) UnmarshalJSON 

func (t *TCPListenPortRange) UnmarshalJSON(b []byte) error

func (*TCPListenPortRange) UnmarshalYAML 

func (t *TCPListenPortRange) UnmarshalYAML(node *yaml.Node) error

type TLSConfig 

type TLSConfig struct {
	tls.Config
}

func (*TLSConfig) UnmarshalJSON 

func (t *TLSConfig) UnmarshalJSON(b []byte) error

type TelemetryOption 

type TelemetryOption struct {
	Jaeger string `yaml:"jaeger" mapstructure:"jaeger"`
}

TelemetryOption is the option for telemetry

type URL 

type URL struct {
	*url.URL
}

URL is simple wrapper around url.URL to make it unmarshallable from a string.

func (*URL) MarshalJSON 

func (u *URL) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaller to print the url.

func (*URL) MarshalYAML 

func (u *URL) MarshalYAML() (interface{}, error)

MarshalYAML implements yaml.Marshaller to print the url.

func (*URL) UnmarshalJSON 

func (u *URL) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler.

func (*URL) UnmarshalYAML 

func (u *URL) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaler.

type UnixListenOption 

type UnixListenOption struct {
	Socket string `mapstructure:"socket" yaml:"socket"`
}

type UploadOption 

type UploadOption struct {
	ListenOption `yaml:",inline" mapstructure:",squash"`
	RateLimit    clientutil.RateLimit `mapstructure:"rateLimit" yaml:"rateLimit"`
}

type WhiteList 

type WhiteList struct {
	Host  string   `yaml:"host" mapstructure:"host"`
	Regx  *Regexp  `yaml:"regx" mapstructure:"regx"`
	Ports []string `yaml:"ports" mapstructure:"ports"`
}

Source Files

  • constants.go
  • constants_otel.go
  • deprecated.go
  • dfget.go
  • dfget_linux.go
  • flags.go
  • headers.go
  • peerhost.go
  • peerhost_linux.go

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.