worker

package
v1.7.7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 6, 2018 License: Apache-2.0 Imports: 62 Imported by: 10

Documentation

Overview

Package worker is a generated protocol buffer package.

It is generated from these files: 

server/worker/worker_service.proto

It has these top-level messages: 

Input
CancelRequest
CancelResponse
ChunkState
Chunks

Index

Constants

View Source 
const (
	// PrometheusPort is the port the aggregated metrics are served on for scraping
	PrometheusPort = 9090
)
View Source 
const (
	// WorkerEtcdPrefix is the prefix in etcd that we use to store worker information.
	WorkerEtcdPrefix = "workers"
)

Variables

View Source 
var (
	ErrInvalidLengthWorkerService = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowWorkerService   = fmt.Errorf("proto: integer overflow")
)
View Source 
var ChunkState_State_name = map[int32]string{
	0: "RUNNING",
	1: "COMPLETE",
	3: "FAILED",
}
View Source 
var ChunkState_State_value = map[string]int32{
	"RUNNING":  0,
	"COMPLETE": 1,
	"FAILED":   3,
}

Functions

func Cancel added in v1.7.5 

func Cancel(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client,
	etcdPrefix string, jobID string, dataFilter []string) error

Cancel cancels a set of datums running on workers. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName.

func Conns added in v1.7.5 

func Conns(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string) ([]*grpc.ClientConn, error)

Conns returns a slice of connections to worker servers. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all clients for all workers.

func HashDatum 

func HashDatum(pipelineName string, pipelineSalt string, data []*Input) string

HashDatum computes and returns the hash of datum + pipeline, with a pipeline-specific prefix.

func HashDatum15 

func HashDatum15(pipelineInfo *pps.PipelineInfo, data []*Input) (string, error)

HashDatum15 computes and returns the hash of datum + pipeline for version <= 1.5.0, with a pipeline-specific prefix.

func MatchDatum 

func MatchDatum(filter []string, data []*pps.InputFile) bool

MatchDatum checks if a datum matches a filter. To match each string in filter must correspond match at least 1 datum's Path or Hash. Order of filter and data is irrelevant.

func RegisterWorkerServer 

func RegisterWorkerServer(s *grpc.Server, srv WorkerServer)

func Status added in v1.7.5 

func Status(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string) ([]*pps.WorkerStatus, error)

Status returns the statuses of workers referenced by pipelineRcName. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all clients for all workers.

Types

type APIServer 

type APIServer struct {
	// contains filtered or unexported fields
}

APIServer implements the worker API

func NewAPIServer 

func NewAPIServer(pachClient *client.APIClient, etcdClient *etcd.Client, etcdPrefix string, pipelineInfo *pps.PipelineInfo, workerName string, namespace string) (*APIServer, error)

NewAPIServer creates an APIServer for a given pipeline

func (*APIServer) Cancel 

func (a *APIServer) Cancel(ctx context.Context, request *CancelRequest) (*CancelResponse, error)

Cancel cancels the currently running datum

func (*APIServer) DatumID added in v1.6.0 

func (a *APIServer) DatumID(data []*Input) string

DatumID computes the id for a datum, this value is used in ListDatum and InspectDatum.

func (*APIServer) Status 

func (a *APIServer) Status(ctx context.Context, _ *types.Empty) (*pps.WorkerStatus, error)

Status returns the status of the current worker.

type CancelRequest 

type CancelRequest struct {
	JobID       string   `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	DataFilters []string `protobuf:"bytes,1,rep,name=data_filters,json=dataFilters" json:"data_filters,omitempty"`
}

func (*CancelRequest) Descriptor 

func (*CancelRequest) Descriptor() ([]byte, []int)

func (*CancelRequest) GetDataFilters 

func (m *CancelRequest) GetDataFilters() []string

func (*CancelRequest) GetJobID 

func (m *CancelRequest) GetJobID() string

func (*CancelRequest) Marshal 

func (m *CancelRequest) Marshal() (dAtA []byte, err error)

func (*CancelRequest) MarshalTo 

func (m *CancelRequest) MarshalTo(dAtA []byte) (int, error)

func (*CancelRequest) ProtoMessage 

func (*CancelRequest) ProtoMessage()

func (*CancelRequest) Reset 

func (m *CancelRequest) Reset()

func (*CancelRequest) Size 

func (m *CancelRequest) Size() (n int)

func (*CancelRequest) String 

func (m *CancelRequest) String() string

func (*CancelRequest) Unmarshal 

func (m *CancelRequest) Unmarshal(dAtA []byte) error

type CancelResponse 

type CancelResponse struct {
	Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
}

func (*CancelResponse) Descriptor 

func (*CancelResponse) Descriptor() ([]byte, []int)

func (*CancelResponse) GetSuccess 

func (m *CancelResponse) GetSuccess() bool

func (*CancelResponse) Marshal 

func (m *CancelResponse) Marshal() (dAtA []byte, err error)

func (*CancelResponse) MarshalTo 

func (m *CancelResponse) MarshalTo(dAtA []byte) (int, error)

func (*CancelResponse) ProtoMessage 

func (*CancelResponse) ProtoMessage()

func (*CancelResponse) Reset 

func (m *CancelResponse) Reset()

func (*CancelResponse) Size 

func (m *CancelResponse) Size() (n int)

func (*CancelResponse) String 

func (m *CancelResponse) String() string

func (*CancelResponse) Unmarshal 

func (m *CancelResponse) Unmarshal(dAtA []byte) error

type ChunkState added in v1.6.6 

type ChunkState struct {
	State   ChunkState_State `protobuf:"varint,1,opt,name=state,proto3,enum=worker.ChunkState_State" json:"state,omitempty"`
	DatumID string           `protobuf:"bytes,2,opt,name=datum_id,json=datumId,proto3" json:"datum_id,omitempty"`
}

func (*ChunkState) Descriptor added in v1.6.6 

func (*ChunkState) Descriptor() ([]byte, []int)

func (*ChunkState) GetDatumID added in v1.6.6 

func (m *ChunkState) GetDatumID() string

func (*ChunkState) GetState added in v1.6.6 

func (m *ChunkState) GetState() ChunkState_State

func (*ChunkState) Marshal added in v1.6.6 

func (m *ChunkState) Marshal() (dAtA []byte, err error)

func (*ChunkState) MarshalTo added in v1.6.6 

func (m *ChunkState) MarshalTo(dAtA []byte) (int, error)

func (*ChunkState) ProtoMessage added in v1.6.6 

func (*ChunkState) ProtoMessage()

func (*ChunkState) Reset added in v1.6.6 

func (m *ChunkState) Reset()

func (*ChunkState) Size added in v1.6.6 

func (m *ChunkState) Size() (n int)

func (*ChunkState) String added in v1.6.6 

func (m *ChunkState) String() string

func (*ChunkState) Unmarshal added in v1.6.6 

func (m *ChunkState) Unmarshal(dAtA []byte) error

type ChunkState_State added in v1.6.6 

type ChunkState_State int32
const (
	ChunkState_RUNNING  ChunkState_State = 0
	ChunkState_COMPLETE ChunkState_State = 1
	ChunkState_FAILED   ChunkState_State = 3
)

func (ChunkState_State) EnumDescriptor added in v1.6.6 

func (ChunkState_State) EnumDescriptor() ([]byte, []int)

func (ChunkState_State) String added in v1.6.6 

func (x ChunkState_State) String() string

type Chunks added in v1.6.6 

type Chunks struct {
	Chunks []int64 `protobuf:"varint,1,rep,packed,name=chunks" json:"chunks,omitempty"`
}

func (*Chunks) Descriptor added in v1.6.6 

func (*Chunks) Descriptor() ([]byte, []int)

func (*Chunks) GetChunks added in v1.6.6 

func (m *Chunks) GetChunks() []int64

func (*Chunks) Marshal added in v1.6.6 

func (m *Chunks) Marshal() (dAtA []byte, err error)

func (*Chunks) MarshalTo added in v1.6.6 

func (m *Chunks) MarshalTo(dAtA []byte) (int, error)

func (*Chunks) ProtoMessage added in v1.6.6 

func (*Chunks) ProtoMessage()

func (*Chunks) Reset added in v1.6.6 

func (m *Chunks) Reset()

func (*Chunks) Size added in v1.6.6 

func (m *Chunks) Size() (n int)

func (*Chunks) String added in v1.6.6 

func (m *Chunks) String() string

func (*Chunks) Unmarshal added in v1.6.6 

func (m *Chunks) Unmarshal(dAtA []byte) error

type Client added in v1.7.5 

type Client struct {
	WorkerClient
	debug.DebugClient
}

Client combines the WorkerAPI and the DebugAPI into a single client.

func Clients added in v1.7.5 

func Clients(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string) ([]Client, error)

Clients returns a slice of worker clients for a pipeline. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all clients for all workers.

type DatumFactory added in v1.5.3 

type DatumFactory interface {
	Len() int
	Datum(i int) []*Input
}

DatumFactory is an interface which allows you to iterate through the datums for a job.

func NewDatumFactory added in v1.5.3 

func NewDatumFactory(pachClient *client.APIClient, input *pps.Input) (DatumFactory, error)

NewDatumFactory creates a datumFactory for an input.

type Input 

type Input struct {
	FileInfo     *pfs.FileInfo `protobuf:"bytes,1,opt,name=file_info,json=fileInfo" json:"file_info,omitempty"`
	ParentCommit *pfs.Commit   `protobuf:"bytes,5,opt,name=parent_commit,json=parentCommit" json:"parent_commit,omitempty"`
	Name         string        `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
	Lazy         bool          `protobuf:"varint,3,opt,name=lazy,proto3" json:"lazy,omitempty"`
	Branch       string        `protobuf:"bytes,4,opt,name=branch,proto3" json:"branch,omitempty"`
	GitURL       string        `protobuf:"bytes,6,opt,name=git_url,json=gitUrl,proto3" json:"git_url,omitempty"`
	EmptyFiles   bool          `protobuf:"varint,7,opt,name=empty_files,json=emptyFiles,proto3" json:"empty_files,omitempty"`
}

func (*Input) Descriptor 

func (*Input) Descriptor() ([]byte, []int)

func (*Input) GetBranch 

func (m *Input) GetBranch() string

func (*Input) GetEmptyFiles added in v1.6.8 

func (m *Input) GetEmptyFiles() bool

func (*Input) GetFileInfo 

func (m *Input) GetFileInfo() *pfs.FileInfo

func (*Input) GetGitURL added in v1.6.4 

func (m *Input) GetGitURL() string

func (*Input) GetLazy 

func (m *Input) GetLazy() bool

func (*Input) GetName 

func (m *Input) GetName() string

func (*Input) GetParentCommit 

func (m *Input) GetParentCommit() *pfs.Commit

func (*Input) Marshal 

func (m *Input) Marshal() (dAtA []byte, err error)

func (*Input) MarshalTo 

func (m *Input) MarshalTo(dAtA []byte) (int, error)

func (*Input) ProtoMessage 

func (*Input) ProtoMessage()

func (*Input) Reset 

func (m *Input) Reset()

func (*Input) Size 

func (m *Input) Size() (n int)

func (*Input) String 

func (m *Input) String() string

func (*Input) Unmarshal 

func (m *Input) Unmarshal(dAtA []byte) error

type WorkerClient 

type WorkerClient interface {
	Status(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*pps.WorkerStatus, error)
	Cancel(ctx context.Context, in *CancelRequest, opts ...grpc.CallOption) (*CancelResponse, error)
}

func NewWorkerClient 

func NewWorkerClient(cc *grpc.ClientConn) WorkerClient

type WorkerServer 

type WorkerServer interface {
	Status(context.Context, *google_protobuf.Empty) (*pps.WorkerStatus, error)
	Cancel(context.Context, *CancelRequest) (*CancelResponse, error)
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.