worker

package
v1.5.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 17, 2017 License: Apache-2.0 Imports: 51 Imported by: 0

Documentation

Overview

Package worker is a generated protocol buffer package.

It is generated from these files: 

server/worker/worker_service.proto

It has these top-level messages: 

Input
ProcessRequest
ProcessResponse
CancelRequest
CancelResponse

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrInvalidLengthWorkerService = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowWorkerService   = fmt.Errorf("proto: integer overflow")
)

Functions

func HashDatum 

func HashDatum(pipelineName string, pipelineSalt string, data []*Input) string

HashDatum computes and returns the hash of datum + pipeline, with a pipeline-specific prefix.

func HashDatum15 

func HashDatum15(pipelineInfo *pps.PipelineInfo, data []*Input) (string, error)

HashDatum15 computes and returns the hash of datum + pipeline for version <= 1.5.0, with a pipeline-specific prefix.

func MatchDatum 

func MatchDatum(filter []string, data []*pps.InputFile) bool

MatchDatum checks if a datum matches a filter. To match each string in filter must correspond match at least 1 datum's Path or Hash. Order of filter and data is irrelevant.

func RegisterWorkerServer 

func RegisterWorkerServer(s *grpc.Server, srv WorkerServer)

Types

type APIServer 

type APIServer struct {
	// contains filtered or unexported fields
}

APIServer implements the worker API

func NewAPIServer 

func NewAPIServer(pachClient *client.APIClient, etcdClient *etcd.Client, etcdPrefix string, pipelineInfo *pps.PipelineInfo, workerName string, namespace string) (*APIServer, error)

NewAPIServer creates an APIServer for a given pipeline

func (*APIServer) Cancel 

func (a *APIServer) Cancel(ctx context.Context, request *CancelRequest) (*CancelResponse, error)

Cancel cancels the currently running datum

func (*APIServer) Process 

func (a *APIServer) Process(ctx context.Context, req *ProcessRequest) (resp *ProcessResponse, retErr error)

Process processes a datum.

func (*APIServer) Status 

func (a *APIServer) Status(ctx context.Context, _ *types.Empty) (*pps.WorkerStatus, error)

Status returns the status of the current worker.

type CancelRequest 

type CancelRequest struct {
	JobID       string   `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	DataFilters []string `protobuf:"bytes,1,rep,name=data_filters,json=dataFilters" json:"data_filters,omitempty"`
}

func (*CancelRequest) Descriptor 

func (*CancelRequest) Descriptor() ([]byte, []int)

func (*CancelRequest) GetDataFilters 

func (m *CancelRequest) GetDataFilters() []string

func (*CancelRequest) GetJobID 

func (m *CancelRequest) GetJobID() string

func (*CancelRequest) Marshal 

func (m *CancelRequest) Marshal() (dAtA []byte, err error)

func (*CancelRequest) MarshalTo 

func (m *CancelRequest) MarshalTo(dAtA []byte) (int, error)

func (*CancelRequest) ProtoMessage 

func (*CancelRequest) ProtoMessage()

func (*CancelRequest) Reset 

func (m *CancelRequest) Reset()

func (*CancelRequest) Size 

func (m *CancelRequest) Size() (n int)

func (*CancelRequest) String 

func (m *CancelRequest) String() string

func (*CancelRequest) Unmarshal 

func (m *CancelRequest) Unmarshal(dAtA []byte) error

type CancelResponse 

type CancelResponse struct {
	Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
}

func (*CancelResponse) Descriptor 

func (*CancelResponse) Descriptor() ([]byte, []int)

func (*CancelResponse) GetSuccess 

func (m *CancelResponse) GetSuccess() bool

func (*CancelResponse) Marshal 

func (m *CancelResponse) Marshal() (dAtA []byte, err error)

func (*CancelResponse) MarshalTo 

func (m *CancelResponse) MarshalTo(dAtA []byte) (int, error)

func (*CancelResponse) ProtoMessage 

func (*CancelResponse) ProtoMessage()

func (*CancelResponse) Reset 

func (m *CancelResponse) Reset()

func (*CancelResponse) Size 

func (m *CancelResponse) Size() (n int)

func (*CancelResponse) String 

func (m *CancelResponse) String() string

func (*CancelResponse) Unmarshal 

func (m *CancelResponse) Unmarshal(dAtA []byte) error

type DatumFactory added in v1.5.3 

type DatumFactory interface {
	Len() int
	Datum(i int) []*Input
}

DatumFactory is an interface which allows you to iterate through the datums for a job.

func NewDatumFactory added in v1.5.3 

func NewDatumFactory(ctx context.Context, pfsClient pfs.APIClient, input *pps.Input) (DatumFactory, error)

NewDatumFactory creates a datumFactory for an input.

type Input 

type Input struct {
	FileInfo     *pfs.FileInfo `protobuf:"bytes,1,opt,name=file_info,json=fileInfo" json:"file_info,omitempty"`
	ParentCommit *pfs.Commit   `protobuf:"bytes,5,opt,name=parent_commit,json=parentCommit" json:"parent_commit,omitempty"`
	Name         string        `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
	Lazy         bool          `protobuf:"varint,3,opt,name=lazy,proto3" json:"lazy,omitempty"`
	Branch       string        `protobuf:"bytes,4,opt,name=branch,proto3" json:"branch,omitempty"`
}

func (*Input) Descriptor 

func (*Input) Descriptor() ([]byte, []int)

func (*Input) GetBranch 

func (m *Input) GetBranch() string

func (*Input) GetFileInfo 

func (m *Input) GetFileInfo() *pfs.FileInfo

func (*Input) GetLazy 

func (m *Input) GetLazy() bool

func (*Input) GetName 

func (m *Input) GetName() string

func (*Input) GetParentCommit 

func (m *Input) GetParentCommit() *pfs.Commit

func (*Input) Marshal 

func (m *Input) Marshal() (dAtA []byte, err error)

func (*Input) MarshalTo 

func (m *Input) MarshalTo(dAtA []byte) (int, error)

func (*Input) ProtoMessage 

func (*Input) ProtoMessage()

func (*Input) Reset 

func (m *Input) Reset()

func (*Input) Size 

func (m *Input) Size() (n int)

func (*Input) String 

func (m *Input) String() string

func (*Input) Unmarshal 

func (m *Input) Unmarshal(dAtA []byte) error

type ProcessRequest 

type ProcessRequest struct {
	// ID of the job for which we're processing 'data'. This is attached to logs
	// generated while processing 'data', so that they can be searched.
	JobID string `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	// The datum to process
	Data []*Input `protobuf:"bytes,1,rep,name=data" json:"data,omitempty"`
	// The tag corresponding to the previous parent's run of this datum, used for
	// incremental jobs, may be nil.
	ParentOutput *pfs.Tag `protobuf:"bytes,3,opt,name=parent_output,json=parentOutput" json:"parent_output,omitempty"`
	EnableStats  bool     `protobuf:"varint,4,opt,name=enable_stats,json=enableStats,proto3" json:"enable_stats,omitempty"`
}

func (*ProcessRequest) Descriptor 

func (*ProcessRequest) Descriptor() ([]byte, []int)

func (*ProcessRequest) GetData 

func (m *ProcessRequest) GetData() []*Input

func (*ProcessRequest) GetEnableStats 

func (m *ProcessRequest) GetEnableStats() bool

func (*ProcessRequest) GetJobID 

func (m *ProcessRequest) GetJobID() string

func (*ProcessRequest) GetParentOutput 

func (m *ProcessRequest) GetParentOutput() *pfs.Tag

func (*ProcessRequest) Marshal 

func (m *ProcessRequest) Marshal() (dAtA []byte, err error)

func (*ProcessRequest) MarshalTo 

func (m *ProcessRequest) MarshalTo(dAtA []byte) (int, error)

func (*ProcessRequest) ProtoMessage 

func (*ProcessRequest) ProtoMessage()

func (*ProcessRequest) Reset 

func (m *ProcessRequest) Reset()

func (*ProcessRequest) Size 

func (m *ProcessRequest) Size() (n int)

func (*ProcessRequest) String 

func (m *ProcessRequest) String() string

func (*ProcessRequest) Unmarshal 

func (m *ProcessRequest) Unmarshal(dAtA []byte) error

type ProcessResponse 

type ProcessResponse struct {
	Stats *pps.ProcessStats `protobuf:"bytes,4,opt,name=stats" json:"stats,omitempty"`
	// If true, the user program has errored
	Failed  bool `protobuf:"varint,2,opt,name=failed,proto3" json:"failed,omitempty"`
	Skipped bool `protobuf:"varint,5,opt,name=skipped,proto3" json:"skipped,omitempty"`
}

ProcessResponse contains a tag, only if the processing was successful.

func (*ProcessResponse) Descriptor 

func (*ProcessResponse) Descriptor() ([]byte, []int)

func (*ProcessResponse) GetFailed 

func (m *ProcessResponse) GetFailed() bool

func (*ProcessResponse) GetSkipped 

func (m *ProcessResponse) GetSkipped() bool

func (*ProcessResponse) GetStats added in v1.5.3 

func (m *ProcessResponse) GetStats() *pps.ProcessStats

func (*ProcessResponse) Marshal 

func (m *ProcessResponse) Marshal() (dAtA []byte, err error)

func (*ProcessResponse) MarshalTo 

func (m *ProcessResponse) MarshalTo(dAtA []byte) (int, error)

func (*ProcessResponse) ProtoMessage 

func (*ProcessResponse) ProtoMessage()

func (*ProcessResponse) Reset 

func (m *ProcessResponse) Reset()

func (*ProcessResponse) Size 

func (m *ProcessResponse) Size() (n int)

func (*ProcessResponse) String 

func (m *ProcessResponse) String() string

func (*ProcessResponse) Unmarshal 

func (m *ProcessResponse) Unmarshal(dAtA []byte) error

type WorkerClient 

type WorkerClient interface {
	Process(ctx context.Context, in *ProcessRequest, opts ...grpc.CallOption) (*ProcessResponse, error)
	Status(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*pps.WorkerStatus, error)
	Cancel(ctx context.Context, in *CancelRequest, opts ...grpc.CallOption) (*CancelResponse, error)
}

func NewWorkerClient 

func NewWorkerClient(cc *grpc.ClientConn) WorkerClient

type WorkerServer 

type WorkerServer interface {
	Process(context.Context, *ProcessRequest) (*ProcessResponse, error)
	Status(context.Context, *google_protobuf.Empty) (*pps.WorkerStatus, error)
	Cancel(context.Context, *CancelRequest) (*CancelResponse, error)
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.