worker

package
v0.4.31 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 15, 2020 License: GPL-3.0 Imports: 92 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultDatabasePath = "/usr/local/share/sonm/geoip/geobase.mmdb"
	DefaultDatabaseURL  = "https://dwh.livenet.sonm.com/geobase.mmdb"
	DefaultDatabaseSHA1 = "2ef2075cc6e4567ab4efca3fde906f4611f18d49"
)

Variables

This section is empty.

Functions

func IsWorkerSSHIdentity added in v0.4.15 

func IsWorkerSSHIdentity(v string) bool

func NewSSHServer added in v0.4.10 

func NewSSHServer(cfg SSHConfig, signer ssh.Signer, credentials *xgrpc.TransportCredentials, sshAuthorization *SSHAuthorization, overseer OverseerView, log *zap.SugaredLogger) (*sshServer, error)

Types

type BackoffTimer 

type BackoffTimer struct {
	// contains filtered or unexported fields
}

BackoffTimer implementation

func NewBackoffTimer 

func NewBackoffTimer(sleep, maxsleep time.Duration) *BackoffTimer

NewBackoffTimer implementations one direction backoff policy

func (*BackoffTimer) C 

func (b *BackoffTimer) C() <-chan time.Time

C resets Timer and returns Timer.C

func (*BackoffTimer) Stop 

func (b *BackoffTimer) Stop() bool

Stop frees the Timer

type BenchmarkHasher 

type BenchmarkHasher interface {
	// HardwareHash returns hash of the hardware, empty string means that we need to rebenchmark everytime
	HardwareHash() string
}

type Config 

type Config struct {
	Endpoint          string                `yaml:"endpoint" required:"true"`
	Logging           logging.Config        `yaml:"logging"`
	Resources         *ResourcesConfig      `yaml:"resources" required:"false"`
	Network           network.NetworkConfig `yaml:"network"`
	Blockchain        *blockchain.Config    `yaml:"blockchain"`
	NPP               npp.Config            `yaml:"npp"`
	SSH               *SSHConfig            `yaml:"ssh" required:"false" `
	PublicIPs         []string              `yaml:"public_ip_addrs" required:"false" `
	Plugins           plugin.Config         `yaml:"plugins"`
	Storage           state.StorageConfig   `yaml:"store"`
	Benchmarks        benchmarks.Config     `yaml:"benchmarks"`
	Whitelist         WhitelistConfig       `yaml:"whitelist"`
	MetricsListenAddr string                `yaml:"metrics_listen_addr" default:"127.0.0.1:14000"`
	DWH               dwh.YAMLConfig        `yaml:"dwh"`
	Matcher           *matcher.YAMLConfig   `yaml:"matcher"`
	Salesman          salesman.YAMLConfig   `yaml:"salesman"`
	Master            common.Address        `yaml:"master" required:"true"`
	Development       DevConfig             `yaml:"development"`
	Admin             *common.Address       `yaml:"admin"`
	MetricsCollector  *common.Address       `yaml:"metrics_collector"`
	Debug             *debug.Config         `yaml:"debug"`
}

func NewConfig 

func NewConfig(path string) (*Config, error)

NewConfig creates a new Worker config from the specified YAML file.

type ContainerInfo 

type ContainerInfo struct {
	ID           string
	ImageName    string
	StartAt      time.Time
	Ports        nat.PortMap
	PublicKey    PublicKey
	Cgroup       string
	CgroupParent string
	NetworkIDs   []string
	DealID       *sonm.BigInt
	TaskId       string
	Tag          *sonm.TaskTag
	AskID        string
	// contains filtered or unexported fields
}

ContainerInfo is a brief information about containers

func (*ContainerInfo) IntoProto 

func (c *ContainerInfo) IntoProto(ctx context.Context) *sonm.TaskStatusReply

type ContainerMetrics 

type ContainerMetrics struct {
	// contains filtered or unexported fields
}

ContainerMetrics are metrics collected from Docker about running containers

func (*ContainerMetrics) Marshal 

func (m *ContainerMetrics) Marshal() *sonm.ResourceUsage

type DealExtractor 

type DealExtractor func(ctx context.Context, request interface{}) (*sonm.BigInt, error)

DealExtractor allows to extract deal id that is used for authorization.

type DealInfoSupplier 

type DealInfoSupplier interface {
	GetDealInfo(ctx context.Context, id *sonm.ID) (*sonm.DealInfoReply, error)
}

type Description 

type Description struct {
	sonm.Container
	Reference    xdocker.Reference
	Auth         string
	Resources    *sonm.AskPlanResources
	CGroupParent string
	Cmd          []string
	TaskId       string
	DealId       string
	Autoremove   bool

	GPUDevices []gpu.GPUID

	NetworkOptions *network.Network
	NetworkSpecs   []*structs.NetworkSpec
	// contains filtered or unexported fields
}

Description for a target application.

func (*Description) DealID added in v0.4.9 

func (d *Description) DealID() string

func (*Description) Expose 

func (d *Description) Expose() (nat.PortSet, nat.PortMap, error)

func (*Description) FormatEnv 

func (d *Description) FormatEnv() []string

func (*Description) GpuDeviceIDs 

func (d *Description) GpuDeviceIDs() []gpu.GPUID

func (*Description) ID 

func (d *Description) ID() string

func (*Description) IsGPURequired 

func (d *Description) IsGPURequired() bool

func (*Description) Mounts 

func (d *Description) Mounts(source string) []volume.Mount

func (*Description) Network added in v0.4.9 

func (d *Description) Network() (string, string)

func (*Description) Networks 

func (d *Description) Networks() []*structs.NetworkSpec

func (*Description) QuotaID added in v0.4.12 

func (d *Description) QuotaID() string

func (*Description) QuotaInBytes added in v0.4.12 

func (d *Description) QuotaInBytes() uint64

func (*Description) Volumes 

func (d *Description) Volumes() map[string]*sonm.Volume

type DevConfig 

type DevConfig struct {
	DisableBenchmarking   bool `yaml:"disable_benchmarking"`
	DisableMasterApproval bool `yaml:"disable_master_approval"`
}

type DeviceKeyer 

type DeviceKeyer interface {
	StorageKey() string
}

type ExecConnection 

type ExecConnection types.HijackedResponse

type GeoIPService added in v0.4.15 

type GeoIPService struct {
	// contains filtered or unexported fields
}

func NewGeoIPService added in v0.4.15 

func NewGeoIPService(cfg *GeoIPServiceConfig) (*GeoIPService, error)

NewGeoIPService constructs a new geo IP service.

func (*GeoIPService) Close added in v0.4.15 

func (m *GeoIPService) Close() error

func (*GeoIPService) Country added in v0.4.15 

func (m *GeoIPService) Country(addr net.IP) (*geoip2.Country, error)

type GeoIPServiceConfig added in v0.4.15 

type GeoIPServiceConfig struct {
	Path string `yaml:"path"`
	URL  string `yaml:"url"`
	SHA1 string `yaml:"sha1"`
}

func (*GeoIPServiceConfig) Normalize added in v0.4.15 

func (m *GeoIPServiceConfig) Normalize() error

type GeoIPServiceUpdater added in v0.4.15 

type GeoIPServiceUpdater struct {
	// contains filtered or unexported fields
}

func (*GeoIPServiceUpdater) Update added in v0.4.15 

func (m *GeoIPServiceUpdater) Update() error

func (*GeoIPServiceUpdater) UpdateIfRequired added in v0.4.15 

func (m *GeoIPServiceUpdater) UpdateIfRequired() error

type NetworkStatsExt added in v0.4.15 

type NetworkStatsExt struct {
	types.NetworkStats

	RxBytesRate   metrics.Meter
	RxPacketsRate metrics.Meter
	RxErrorsRate  metrics.Meter
	RxDroppedRate metrics.Meter
	TxBytesRate   metrics.Meter
	TxPacketsRate metrics.Meter
	TxErrorsRate  metrics.Meter
	TxDroppedRate metrics.Meter
}

NetworkStatsExt extends the Docker "NetworkStats" structure by adding rate meters.

func NewNetworkStatsExt added in v0.4.15 

func NewNetworkStatsExt(v types.NetworkStats) *NetworkStatsExt

NewNetworkStatsExt constructs a new extended network stats, initializing with the given argument.

func (*NetworkStatsExt) Update added in v0.4.15 

func (m *NetworkStatsExt) Update(v types.NetworkStats)

Update updates the current extended network stats, calculating rates.

type Option 

type Option func(*options)

func WithContext 

func WithContext(ctx context.Context) Option

func WithLogWatcher added in v0.4.21 

func WithLogWatcher(watcher *logging.WatcherCore) Option

func WithVersion 

func WithVersion(v string) Option

type Overseer 

type Overseer interface {
	// Load loads an image from the specified reader to the Docker.
	Load(ctx context.Context, rd io.Reader) (imageID, error)

	// Save saves an image from the Docker into the returned reader.
	Save(ctx context.Context, imageID string) (types.ImageInspect, io.ReadCloser, error)

	// Spool prepares an application for its further start.
	//
	// For Docker containers this is an equivalent of pulling from the registry.
	Spool(ctx context.Context, d Description) error

	// Start attempts to start an application using the specified description.
	//
	// After successful starting an application becomes a target for accepting request, but not guarantees
	// to complete them.
	Start(ctx context.Context, description Description) (chan sonm.TaskStatusReply_Status, ContainerInfo, error)

	// Attach attemps to attach to a running application with a specified description
	Attach(ctx context.Context, ID string, description Description) (chan sonm.TaskStatusReply_Status, error)

	// Exec a given command in running container
	Exec(ctx context.Context, Id string, cmd []string, env []string, isTty bool, wCh <-chan ssh.Window) (types.HijackedResponse, error)

	// Stop terminates the container.
	Stop(ctx context.Context, containerID string) error

	// OnDealFinish makes all cleanup related to closed deal
	OnDealFinish(ctx context.Context, containerID string) error

	// Info returns runtime statistics collected from all running containers.
	//
	// Depending on the implementation this can be cached.
	Info(ctx context.Context) (map[string]ContainerMetrics, error)

	// Logs fetch logs of the container
	Logs(ctx context.Context, id string, opts types.ContainerLogsOptions) (io.ReadCloser, error)

	// Close terminates all associated asynchronous operations and prepares the Overseer for shutting down.
	Close() error
}

Overseer watches all worker's applications.

func NewOverseer 

func NewOverseer(ctx context.Context, plugins *plugin.Repository) (Overseer, error)

NewOverseer creates new overseer

type OverseerView added in v0.4.10 

type OverseerView interface {
	ContainerInfo(id string) (*ContainerInfo, bool)
	// ConsumerIdentityLevel returns the consumer identity level by the given
	// task identifier.
	ConsumerIdentityLevel(ctx context.Context, id string) (sonm.IdentityLevel, error)
	ExecIdentity() sonm.IdentityLevel
	Exec(ctx context.Context, id string, cmd []string, env []string, isTty bool, wCh <-chan sshd.Window) (types.HijackedResponse, error)
}

OverseerView is a bridge between keeping "Worker" as a parameter and slightly more decomposed architecture.

type PublicKey added in v0.4.11 

type PublicKey struct {
	ssh.PublicKey
}

func (PublicKey) MarshalText added in v0.4.11 

func (m PublicKey) MarshalText() ([]byte, error)

func (*PublicKey) UnmarshalText added in v0.4.11 

func (m *PublicKey) UnmarshalText(data []byte) error

type ResourcesConfig 

type ResourcesConfig struct {
	Cgroup    string                `required:"true" yaml:"cgroup"`
	Resources *specs.LinuxResources `required:"false" yaml:"resources"`
}

type SSH 

type SSH interface {
	Run(ctx context.Context) error
	Close() error
}

type SSHAuthorization added in v0.4.15 

type SSHAuthorization struct {
	// contains filtered or unexported fields
}

func NewSSHAuthorization added in v0.4.15 

func NewSSHAuthorization() *SSHAuthorization

func (*SSHAuthorization) Allow added in v0.4.15 

func (m *SSHAuthorization) Allow(key common.Address, options ...SSHAuthorizationOption)

Allow adds the given key to the whitelist.

func (*SSHAuthorization) Deny added in v0.4.15 

func (m *SSHAuthorization) Deny(key common.Address, options ...SSHAuthorizationOption)

Deny adds the given key to the blacklist.

func (*SSHAuthorization) IsAllowed added in v0.4.15 

func (m *SSHAuthorization) IsAllowed(key common.Address) bool

IsAllowed returns true if the given key passes the authorization.

type SSHAuthorizationOption added in v0.4.15 

type SSHAuthorizationOption func(options *sshAuthorizationOptions)

func WithExpiration added in v0.4.15 

func WithExpiration(duration time.Duration) SSHAuthorizationOption

type SSHConfig 

type SSHConfig struct {
	Endpoint  string             `yaml:"endpoint" default:":0"`
	NPP       npp.Config         `yaml:"npp"`
	Identity  sonm.IdentityLevel `yaml:"identity" default:"identified"`
	Blacklist []common.Address   `yaml:"blacklist"`
}

type Whitelist 

type Whitelist interface {
	Allowed(ctx context.Context, ref xdocker.Reference, auth string) (bool, xdocker.Reference, error)
}

func NewWhitelist 

func NewWhitelist(ctx context.Context, config *WhitelistConfig) Whitelist

type WhitelistConfig 

type WhitelistConfig struct {
	Url string `yaml:"url"`
	// Deprecated: use PrivilegedIdentityLevel instead. Breaking issue #1470.
	Enabled                 *bool              `yaml:"enabled" default:"true" required:"true"`
	PrivilegedAddresses     []string           `yaml:"privileged_addresses"`
	RefreshPeriod           uint               `yaml:"refresh_period" default:"60"`
	PrivilegedIdentityLevel sonm.IdentityLevel `yaml:"privileged_identity_level" default:"identified"`
}

type WhitelistRecord 

type WhitelistRecord struct {
	AllowedHashes []digest.Digest `json:"allowed_hashes"`
}

type Worker 

type Worker struct {

	// Embedded inspection service.
	*inspect.InspectService
	// contains filtered or unexported fields
}

Worker holds information about jobs, make orders to Observer and communicates with Worker

func NewWorker 

func NewWorker(cfg *Config, storage *state.Storage, options ...Option) (*Worker, error)

func (*Worker) AddCapability added in v0.4.21 

func (m *Worker) AddCapability(ctx context.Context, request *sonm.WorkerAddCapabilityRequest) (*sonm.WorkerAddCapabilityResponse, error)

func (*Worker) AskPlanByTaskID 

func (m *Worker) AskPlanByTaskID(taskID string) (*sonm.AskPlan, error)

func (*Worker) AskPlans 

func (m *Worker) AskPlans(ctx context.Context, _ *sonm.Empty) (*sonm.AskPlansReply, error)

func (*Worker) CancelDealTasks added in v0.4.13 

func (m *Worker) CancelDealTasks(ctx context.Context, dealID *sonm.BigInt) error

func (*Worker) Close 

func (m *Worker) Close()

Close disposes all resources related to the Worker

func (*Worker) CollectTasksStatuses 

func (m *Worker) CollectTasksStatuses(statuses ...sonm.TaskStatusReply_Status) map[string]*sonm.TaskStatusReply

func (*Worker) CreateAskPlan 

func (m *Worker) CreateAskPlan(ctx context.Context, request *sonm.AskPlan) (*sonm.ID, error)

func (*Worker) DebugState 

func (m *Worker) DebugState(ctx context.Context, _ *sonm.Empty) (*sonm.DebugStateReply, error)

func (*Worker) Devices 

func (m *Worker) Devices(ctx context.Context, request *sonm.Empty) (*sonm.DevicesReply, error)

func (*Worker) FreeDevices 

func (m *Worker) FreeDevices(ctx context.Context, request *sonm.Empty) (*sonm.DevicesReply, error)

FreeDevices provides information about unallocated resources that can be turned into ask-plans. Deprecated: no longer usable

func (*Worker) GetContainerInfo 

func (m *Worker) GetContainerInfo(id string) (*ContainerInfo, bool)

func (*Worker) GetDealInfo 

func (m *Worker) GetDealInfo(ctx context.Context, id *sonm.ID) (*sonm.DealInfoReply, error)

func (*Worker) JoinNetwork 

func (m *Worker) JoinNetwork(ctx context.Context, request *sonm.WorkerJoinNetworkRequest) (*sonm.NetworkSpec, error)

TODO: proper request

func (*Worker) Metrics added in v0.4.17 

func (m *Worker) Metrics(ctx context.Context, req *sonm.WorkerMetricsRequest) (*sonm.WorkerMetricsResponse, error)

func (*Worker) NextMaintenance 

func (m *Worker) NextMaintenance(ctx context.Context, _ *sonm.Empty) (*sonm.Timestamp, error)

func (*Worker) PullTask 

func (m *Worker) PullTask(request *sonm.PullTaskRequest, stream sonm.Worker_PullTaskServer) error

func (*Worker) PurgeAskPlans 

func (m *Worker) PurgeAskPlans(ctx context.Context, _ *sonm.Empty) (*sonm.Empty, error)

func (*Worker) PurgeAskPlansDetailed added in v0.4.13 

func (m *Worker) PurgeAskPlansDetailed(ctx context.Context, _ *sonm.Empty) (*sonm.ErrorByStringID, error)

func (*Worker) PurgeBenchmarks 

func (m *Worker) PurgeBenchmarks(ctx context.Context, _ *sonm.Empty) (*sonm.Empty, error)

func (*Worker) PurgeTasks added in v0.4.13 

func (m *Worker) PurgeTasks(ctx context.Context, request *sonm.PurgeTasksRequest) (*sonm.ErrorByStringID, error)

func (*Worker) PushTask 

func (m *Worker) PushTask(stream sonm.Worker_PushTaskServer) error

func (*Worker) RemoveAskPlan 

func (m *Worker) RemoveAskPlan(ctx context.Context, request *sonm.ID) (*sonm.Empty, error)

func (*Worker) RemoveBenchmark 

func (m *Worker) RemoveBenchmark(ctx context.Context, id *sonm.NumericID) (*sonm.Empty, error)

func (*Worker) RemoveCapability added in v0.4.21 

func (m *Worker) RemoveCapability(ctx context.Context, request *sonm.WorkerRemoveCapabilityRequest) (*sonm.WorkerRemoveCapabilityResponse, error)

func (*Worker) RunSSH 

func (m *Worker) RunSSH(ctx context.Context) error

func (*Worker) ScheduleMaintenance 

func (m *Worker) ScheduleMaintenance(ctx context.Context, timestamp *sonm.Timestamp) (*sonm.Empty, error)

func (*Worker) Serve 

func (m *Worker) Serve() error

Serve starts handling incoming API gRPC requests

func (*Worker) StartTask 

func (m *Worker) StartTask(ctx context.Context, request *sonm.StartTaskRequest) (*sonm.StartTaskReply, error)

func (*Worker) Status 

func (m *Worker) Status(ctx context.Context, _ *sonm.Empty) (*sonm.StatusReply, error)

Status returns internal worker statistic

func (*Worker) StopTask 

func (m *Worker) StopTask(ctx context.Context, request *sonm.ID) (*sonm.Empty, error)

StopTask request forces to kill container

func (*Worker) TaskLogs 

func (m *Worker) TaskLogs(request *sonm.TaskLogsRequest, server sonm.Worker_TaskLogsServer) error

TaskLogs returns logs from container

func (*Worker) TaskStatus 

func (m *Worker) TaskStatus(ctx context.Context, req *sonm.ID) (*sonm.TaskStatusReply, error)

func (*Worker) Tasks 

func (m *Worker) Tasks(ctx context.Context, request *sonm.Empty) (*sonm.TaskListReply, error)

Source Files

View all Source files

Directories

Path Synopsis
tc
btrfs

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.