Documentation ¶
Index ¶
- Variables
- func ComposeFileSets(pachClient *client.APIClient, taskDoer task.Doer, fileSetIDs []string) (string, error)
- func Create(pachClient *client.APIClient, taskDoer task.Doer, input *pps.Input) (string, error)
- func CreateEmptyFileSet(pachClient *client.APIClient) (string, error)
- func CreateSets(pachClient *client.APIClient, setSpec *SetSpec, fileSetID string, ...) ([]*pfs.PathRange, error)
- func IsTask(input *anypb.Any) bool
- func IsTaskResult(output *anypb.Any) bool
- func Merge(dits []Iterator, cb func([]*Meta) error) error
- func MergeProcessStats(x, y *pps.ProcessStats)
- func MergeStats(x, y *Stats)
- func ProcessTask(pachClient *client.APIClient, input *anypb.Any) (*anypb.Any, error)
- func TaskResultFileSets(output *anypb.Any) ([]string, error)
- func WithCreateFileSet(pachClient *client.APIClient, name string, cb func(*Set) error) (string, error)
- func WithSet(cacheClient *pfssync.CacheClient, storageRoot string, cb func(*Set) error, ...) (retErr error)
- type ComposeTask
- func (*ComposeTask) Descriptor() ([]byte, []int)deprecated
- func (x *ComposeTask) GetAuthToken() string
- func (x *ComposeTask) GetFileSetIds() []string
- func (x *ComposeTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*ComposeTask) ProtoMessage()
- func (x *ComposeTask) ProtoReflect() protoreflect.Message
- func (x *ComposeTask) Reset()
- func (x *ComposeTask) String() string
- type ComposeTaskResult
- func (*ComposeTaskResult) Descriptor() ([]byte, []int)deprecated
- func (x *ComposeTaskResult) GetFileSetId() string
- func (x *ComposeTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*ComposeTaskResult) ProtoMessage()
- func (x *ComposeTaskResult) ProtoReflect() protoreflect.Message
- func (x *ComposeTaskResult) Reset()
- func (x *ComposeTaskResult) String() string
- type CrossTask
- func (*CrossTask) Descriptor() ([]byte, []int)deprecated
- func (x *CrossTask) GetAuthToken() string
- func (x *CrossTask) GetBaseFileSetIndex() int64
- func (x *CrossTask) GetBaseFileSetPathRange() *pfs.PathRange
- func (x *CrossTask) GetBaseIndex() int64
- func (x *CrossTask) GetFileSetIds() []string
- func (x *CrossTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*CrossTask) ProtoMessage()
- func (x *CrossTask) ProtoReflect() protoreflect.Message
- func (x *CrossTask) Reset()
- func (x *CrossTask) String() string
- type CrossTaskResult
- func (*CrossTaskResult) Descriptor() ([]byte, []int)deprecated
- func (x *CrossTaskResult) GetFileSetId() string
- func (x *CrossTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*CrossTaskResult) ProtoMessage()
- func (x *CrossTaskResult) ProtoReflect() protoreflect.Message
- func (x *CrossTaskResult) Reset()
- func (x *CrossTaskResult) String() string
- type Datum
- type Deleter
- type Hasher
- type Iterator
- func NewCommitIterator(pachClient *client.APIClient, commit *pfs.Commit, pathRange *pfs.PathRange) Iterator
- func NewFileSetIterator(pachClient *client.APIClient, fsID string, pathRange *pfs.PathRange) Iterator
- func NewIterator(pachClient *client.APIClient, taskDoer task.Doer, input *pps.Input) (Iterator, error)
- func NewJobIterator(iterator Iterator, job *pps.Job, hasher Hasher) Iterator
- type KeyTask
- func (*KeyTask) Descriptor() ([]byte, []int)deprecated
- func (x *KeyTask) GetAuthToken() string
- func (x *KeyTask) GetFileSetId() string
- func (x *KeyTask) GetPathRange() *pfs.PathRange
- func (x *KeyTask) GetType() KeyTask_Type
- func (x *KeyTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*KeyTask) ProtoMessage()
- func (x *KeyTask) ProtoReflect() protoreflect.Message
- func (x *KeyTask) Reset()
- func (x *KeyTask) String() string
- type KeyTaskResult
- func (*KeyTaskResult) Descriptor() ([]byte, []int)deprecated
- func (x *KeyTaskResult) GetFileSetId() string
- func (x *KeyTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*KeyTaskResult) ProtoMessage()
- func (x *KeyTaskResult) ProtoReflect() protoreflect.Message
- func (x *KeyTaskResult) Reset()
- func (x *KeyTaskResult) String() string
- type KeyTask_Type
- func (KeyTask_Type) Descriptor() protoreflect.EnumDescriptor
- func (x KeyTask_Type) Enum() *KeyTask_Type
- func (KeyTask_Type) EnumDescriptor() ([]byte, []int)deprecated
- func (x KeyTask_Type) Number() protoreflect.EnumNumber
- func (x KeyTask_Type) String() string
- func (KeyTask_Type) Type() protoreflect.EnumType
- type MergeTask
- func (*MergeTask) Descriptor() ([]byte, []int)deprecated
- func (x *MergeTask) GetAuthToken() string
- func (x *MergeTask) GetFileSetIds() []string
- func (x *MergeTask) GetPathRange() *pfs.PathRange
- func (x *MergeTask) GetType() MergeTask_Type
- func (x *MergeTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*MergeTask) ProtoMessage()
- func (x *MergeTask) ProtoReflect() protoreflect.Message
- func (x *MergeTask) Reset()
- func (x *MergeTask) String() string
- type MergeTaskResult
- func (*MergeTaskResult) Descriptor() ([]byte, []int)deprecated
- func (x *MergeTaskResult) GetFileSetId() string
- func (x *MergeTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*MergeTaskResult) ProtoMessage()
- func (x *MergeTaskResult) ProtoReflect() protoreflect.Message
- func (x *MergeTaskResult) Reset()
- func (x *MergeTaskResult) String() string
- type MergeTask_Type
- func (MergeTask_Type) Descriptor() protoreflect.EnumDescriptor
- func (x MergeTask_Type) Enum() *MergeTask_Type
- func (MergeTask_Type) EnumDescriptor() ([]byte, []int)deprecated
- func (x MergeTask_Type) Number() protoreflect.EnumNumber
- func (x MergeTask_Type) String() string
- func (MergeTask_Type) Type() protoreflect.EnumType
- type Meta
- func (*Meta) Descriptor() ([]byte, []int)deprecated
- func (x *Meta) GetHash() string
- func (x *Meta) GetImageId() string
- func (x *Meta) GetIndex() int64
- func (x *Meta) GetInputs() []*common.Input
- func (x *Meta) GetJob() *pps.Job
- func (x *Meta) GetReason() string
- func (x *Meta) GetState() State
- func (x *Meta) GetStats() *pps.ProcessStats
- func (x *Meta) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*Meta) ProtoMessage()
- func (x *Meta) ProtoReflect() protoreflect.Message
- func (x *Meta) Reset()
- func (x *Meta) String() string
- type Option
- type PFSTask
- func (*PFSTask) Descriptor() ([]byte, []int)deprecated
- func (x *PFSTask) GetAuthToken() string
- func (x *PFSTask) GetBaseIndex() int64
- func (x *PFSTask) GetInput() *pps.PFSInput
- func (x *PFSTask) GetPathRange() *pfs.PathRange
- func (x *PFSTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*PFSTask) ProtoMessage()
- func (x *PFSTask) ProtoReflect() protoreflect.Message
- func (x *PFSTask) Reset()
- func (x *PFSTask) String() string
- type PFSTaskResult
- func (*PFSTaskResult) Descriptor() ([]byte, []int)deprecated
- func (x *PFSTaskResult) GetFileSetId() string
- func (x *PFSTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*PFSTaskResult) ProtoMessage()
- func (x *PFSTaskResult) ProtoReflect() protoreflect.Message
- func (x *PFSTaskResult) Reset()
- func (x *PFSTaskResult) String() string
- type Set
- type SetOption
- type SetSpec
- func (*SetSpec) Descriptor() ([]byte, []int)deprecated
- func (x *SetSpec) GetNumber() int64
- func (x *SetSpec) GetSizeBytes() int64
- func (x *SetSpec) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*SetSpec) ProtoMessage()
- func (x *SetSpec) ProtoReflect() protoreflect.Message
- func (x *SetSpec) Reset()
- func (x *SetSpec) String() string
- type State
- type Stats
- func (*Stats) Descriptor() ([]byte, []int)deprecated
- func (x *Stats) GetFailed() int64
- func (x *Stats) GetFailedId() string
- func (x *Stats) GetProcessStats() *pps.ProcessStats
- func (x *Stats) GetProcessed() int64
- func (x *Stats) GetRecovered() int64
- func (x *Stats) GetSkipped() int64
- func (x *Stats) GetTotal() int64
- func (x *Stats) MarshalLogObject(enc zapcore.ObjectEncoder) error
- func (*Stats) ProtoMessage()
- func (x *Stats) ProtoReflect() protoreflect.Message
- func (x *Stats) Reset()
- func (x *Stats) String() string
Constants ¶
This section is empty.
Variables ¶
var ( State_name = map[int32]string{ 0: "PROCESSED", 1: "FAILED", 2: "RECOVERED", } State_value = map[string]int32{ "PROCESSED": 0, "FAILED": 1, "RECOVERED": 2, } )
Enum value maps for State.
var ( KeyTask_Type_name = map[int32]string{ 0: "JOIN", 1: "GROUP", } KeyTask_Type_value = map[string]int32{ "JOIN": 0, "GROUP": 1, } )
Enum value maps for KeyTask_Type.
var ( MergeTask_Type_name = map[int32]string{ 0: "JOIN", 1: "GROUP", } MergeTask_Type_value = map[string]int32{ "JOIN": 0, "GROUP": 1, } )
Enum value maps for MergeTask_Type.
var File_server_worker_datum_datum_proto protoreflect.FileDescriptor
Functions ¶
func ComposeFileSets ¶
func CreateSets ¶
func IsTaskResult ¶
func MergeProcessStats ¶
func MergeProcessStats(x, y *pps.ProcessStats)
MergeProcessStats merges two process stats.
func ProcessTask ¶
func WithCreateFileSet ¶
Types ¶
type ComposeTask ¶
type ComposeTask struct { FileSetIds []string `protobuf:"bytes,1,rep,name=file_set_ids,json=fileSetIds,proto3" json:"file_set_ids,omitempty"` AuthToken string `protobuf:"bytes,2,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"` // contains filtered or unexported fields }
func (*ComposeTask) Descriptor
deprecated
func (*ComposeTask) Descriptor() ([]byte, []int)
Deprecated: Use ComposeTask.ProtoReflect.Descriptor instead.
func (*ComposeTask) GetAuthToken ¶
func (x *ComposeTask) GetAuthToken() string
func (*ComposeTask) GetFileSetIds ¶
func (x *ComposeTask) GetFileSetIds() []string
func (*ComposeTask) MarshalLogObject ¶
func (x *ComposeTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*ComposeTask) ProtoMessage ¶
func (*ComposeTask) ProtoMessage()
func (*ComposeTask) ProtoReflect ¶ added in v2.7.0
func (x *ComposeTask) ProtoReflect() protoreflect.Message
func (*ComposeTask) Reset ¶
func (x *ComposeTask) Reset()
func (*ComposeTask) String ¶
func (x *ComposeTask) String() string
type ComposeTaskResult ¶
type ComposeTaskResult struct { FileSetId string `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"` // contains filtered or unexported fields }
func (*ComposeTaskResult) Descriptor
deprecated
func (*ComposeTaskResult) Descriptor() ([]byte, []int)
Deprecated: Use ComposeTaskResult.ProtoReflect.Descriptor instead.
func (*ComposeTaskResult) GetFileSetId ¶
func (x *ComposeTaskResult) GetFileSetId() string
func (*ComposeTaskResult) MarshalLogObject ¶
func (x *ComposeTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*ComposeTaskResult) ProtoMessage ¶
func (*ComposeTaskResult) ProtoMessage()
func (*ComposeTaskResult) ProtoReflect ¶ added in v2.7.0
func (x *ComposeTaskResult) ProtoReflect() protoreflect.Message
func (*ComposeTaskResult) Reset ¶
func (x *ComposeTaskResult) Reset()
func (*ComposeTaskResult) String ¶
func (x *ComposeTaskResult) String() string
type CrossTask ¶
type CrossTask struct { FileSetIds []string `protobuf:"bytes,1,rep,name=file_set_ids,json=fileSetIds,proto3" json:"file_set_ids,omitempty"` BaseFileSetIndex int64 `protobuf:"varint,2,opt,name=base_file_set_index,json=baseFileSetIndex,proto3" json:"base_file_set_index,omitempty"` BaseFileSetPathRange *pfs.PathRange `` /* 127-byte string literal not displayed */ BaseIndex int64 `protobuf:"varint,4,opt,name=base_index,json=baseIndex,proto3" json:"base_index,omitempty"` AuthToken string `protobuf:"bytes,5,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"` // contains filtered or unexported fields }
func (*CrossTask) Descriptor
deprecated
func (*CrossTask) GetAuthToken ¶
func (*CrossTask) GetBaseFileSetIndex ¶
func (*CrossTask) GetBaseFileSetPathRange ¶
func (*CrossTask) GetBaseIndex ¶
func (*CrossTask) GetFileSetIds ¶
func (*CrossTask) MarshalLogObject ¶
func (x *CrossTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*CrossTask) ProtoMessage ¶
func (*CrossTask) ProtoMessage()
func (*CrossTask) ProtoReflect ¶ added in v2.7.0
func (x *CrossTask) ProtoReflect() protoreflect.Message
type CrossTaskResult ¶
type CrossTaskResult struct { FileSetId string `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"` // contains filtered or unexported fields }
func (*CrossTaskResult) Descriptor
deprecated
func (*CrossTaskResult) Descriptor() ([]byte, []int)
Deprecated: Use CrossTaskResult.ProtoReflect.Descriptor instead.
func (*CrossTaskResult) GetFileSetId ¶
func (x *CrossTaskResult) GetFileSetId() string
func (*CrossTaskResult) MarshalLogObject ¶
func (x *CrossTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*CrossTaskResult) ProtoMessage ¶
func (*CrossTaskResult) ProtoMessage()
func (*CrossTaskResult) ProtoReflect ¶ added in v2.7.0
func (x *CrossTaskResult) ProtoReflect() protoreflect.Message
func (*CrossTaskResult) Reset ¶
func (x *CrossTaskResult) Reset()
func (*CrossTaskResult) String ¶
func (x *CrossTaskResult) String() string
type Datum ¶
Datum manages a datum.
func (*Datum) MetaStorageRoot ¶
MetaStorageRoot returns the meta storage root.
func (*Datum) PFSStorageRoot ¶
PFSStorageRoot returns the pfs storage root.
type Deleter ¶
Deleter deletes a datum.
func NewDeleter ¶
func NewDeleter(metaFileWalker fileWalkerFunc, metaOutputClient, pfsOutputClient client.ModifyFile) Deleter
NewDeleter creates a new deleter.
type Hasher ¶
type Hasher interface { // Hash computes the datum hash based on the inputs. Hash([]*common.Input) string }
Hasher is the standard interface for a datum hasher.
type Iterator ¶
type Iterator interface { // Iterate iterates over a set of datums. Iterate(func(*Meta) error) error }
Iterator is the standard interface for a datum iterator.
func NewCommitIterator ¶
func NewCommitIterator(pachClient *client.APIClient, commit *pfs.Commit, pathRange *pfs.PathRange) Iterator
NewCommitIterator creates an iterator for the specified commit and repo.
func NewFileSetIterator ¶
func NewFileSetIterator(pachClient *client.APIClient, fsID string, pathRange *pfs.PathRange) Iterator
NewFileSetIterator creates a new fileset iterator.
type KeyTask ¶
type KeyTask struct { FileSetId string `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"` PathRange *pfs.PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"` Type KeyTask_Type `protobuf:"varint,3,opt,name=type,proto3,enum=datum.KeyTask_Type" json:"type,omitempty"` AuthToken string `protobuf:"bytes,4,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"` // contains filtered or unexported fields }
func (*KeyTask) Descriptor
deprecated
func (*KeyTask) GetAuthToken ¶
func (*KeyTask) GetFileSetId ¶
func (*KeyTask) GetPathRange ¶
func (*KeyTask) GetType ¶
func (x *KeyTask) GetType() KeyTask_Type
func (*KeyTask) MarshalLogObject ¶
func (x *KeyTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*KeyTask) ProtoMessage ¶
func (*KeyTask) ProtoMessage()
func (*KeyTask) ProtoReflect ¶ added in v2.7.0
func (x *KeyTask) ProtoReflect() protoreflect.Message
type KeyTaskResult ¶
type KeyTaskResult struct { FileSetId string `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"` // contains filtered or unexported fields }
func (*KeyTaskResult) Descriptor
deprecated
func (*KeyTaskResult) Descriptor() ([]byte, []int)
Deprecated: Use KeyTaskResult.ProtoReflect.Descriptor instead.
func (*KeyTaskResult) GetFileSetId ¶
func (x *KeyTaskResult) GetFileSetId() string
func (*KeyTaskResult) MarshalLogObject ¶
func (x *KeyTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*KeyTaskResult) ProtoMessage ¶
func (*KeyTaskResult) ProtoMessage()
func (*KeyTaskResult) ProtoReflect ¶ added in v2.7.0
func (x *KeyTaskResult) ProtoReflect() protoreflect.Message
func (*KeyTaskResult) Reset ¶
func (x *KeyTaskResult) Reset()
func (*KeyTaskResult) String ¶
func (x *KeyTaskResult) String() string
type KeyTask_Type ¶
type KeyTask_Type int32
const ( KeyTask_JOIN KeyTask_Type = 0 KeyTask_GROUP KeyTask_Type = 1 )
func (KeyTask_Type) Descriptor ¶ added in v2.7.0
func (KeyTask_Type) Descriptor() protoreflect.EnumDescriptor
func (KeyTask_Type) Enum ¶ added in v2.7.0
func (x KeyTask_Type) Enum() *KeyTask_Type
func (KeyTask_Type) EnumDescriptor
deprecated
func (KeyTask_Type) EnumDescriptor() ([]byte, []int)
Deprecated: Use KeyTask_Type.Descriptor instead.
func (KeyTask_Type) Number ¶ added in v2.7.0
func (x KeyTask_Type) Number() protoreflect.EnumNumber
func (KeyTask_Type) String ¶
func (x KeyTask_Type) String() string
func (KeyTask_Type) Type ¶ added in v2.7.0
func (KeyTask_Type) Type() protoreflect.EnumType
type MergeTask ¶
type MergeTask struct { FileSetIds []string `protobuf:"bytes,1,rep,name=file_set_ids,json=fileSetIds,proto3" json:"file_set_ids,omitempty"` PathRange *pfs.PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"` Type MergeTask_Type `protobuf:"varint,3,opt,name=type,proto3,enum=datum.MergeTask_Type" json:"type,omitempty"` AuthToken string `protobuf:"bytes,4,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"` // contains filtered or unexported fields }
func (*MergeTask) Descriptor
deprecated
func (*MergeTask) GetAuthToken ¶
func (*MergeTask) GetFileSetIds ¶
func (*MergeTask) GetPathRange ¶
func (*MergeTask) GetType ¶
func (x *MergeTask) GetType() MergeTask_Type
func (*MergeTask) MarshalLogObject ¶
func (x *MergeTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*MergeTask) ProtoMessage ¶
func (*MergeTask) ProtoMessage()
func (*MergeTask) ProtoReflect ¶ added in v2.7.0
func (x *MergeTask) ProtoReflect() protoreflect.Message
type MergeTaskResult ¶
type MergeTaskResult struct { FileSetId string `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"` // contains filtered or unexported fields }
func (*MergeTaskResult) Descriptor
deprecated
func (*MergeTaskResult) Descriptor() ([]byte, []int)
Deprecated: Use MergeTaskResult.ProtoReflect.Descriptor instead.
func (*MergeTaskResult) GetFileSetId ¶
func (x *MergeTaskResult) GetFileSetId() string
func (*MergeTaskResult) MarshalLogObject ¶
func (x *MergeTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*MergeTaskResult) ProtoMessage ¶
func (*MergeTaskResult) ProtoMessage()
func (*MergeTaskResult) ProtoReflect ¶ added in v2.7.0
func (x *MergeTaskResult) ProtoReflect() protoreflect.Message
func (*MergeTaskResult) Reset ¶
func (x *MergeTaskResult) Reset()
func (*MergeTaskResult) String ¶
func (x *MergeTaskResult) String() string
type MergeTask_Type ¶
type MergeTask_Type int32
const ( MergeTask_JOIN MergeTask_Type = 0 MergeTask_GROUP MergeTask_Type = 1 )
func (MergeTask_Type) Descriptor ¶ added in v2.7.0
func (MergeTask_Type) Descriptor() protoreflect.EnumDescriptor
func (MergeTask_Type) Enum ¶ added in v2.7.0
func (x MergeTask_Type) Enum() *MergeTask_Type
func (MergeTask_Type) EnumDescriptor
deprecated
func (MergeTask_Type) EnumDescriptor() ([]byte, []int)
Deprecated: Use MergeTask_Type.Descriptor instead.
func (MergeTask_Type) Number ¶ added in v2.7.0
func (x MergeTask_Type) Number() protoreflect.EnumNumber
func (MergeTask_Type) String ¶
func (x MergeTask_Type) String() string
func (MergeTask_Type) Type ¶ added in v2.7.0
func (MergeTask_Type) Type() protoreflect.EnumType
type Meta ¶
type Meta struct { Job *pps.Job `protobuf:"bytes,1,opt,name=job,proto3" json:"job,omitempty"` Inputs []*common.Input `protobuf:"bytes,2,rep,name=inputs,proto3" json:"inputs,omitempty"` Hash string `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"` State State `protobuf:"varint,4,opt,name=state,proto3,enum=datum.State" json:"state,omitempty"` Reason string `protobuf:"bytes,5,opt,name=reason,proto3" json:"reason,omitempty"` Stats *pps.ProcessStats `protobuf:"bytes,6,opt,name=stats,proto3" json:"stats,omitempty"` Index int64 `protobuf:"varint,7,opt,name=index,proto3" json:"index,omitempty"` ImageId string `protobuf:"bytes,8,opt,name=image_id,json=imageId,proto3" json:"image_id,omitempty"` // contains filtered or unexported fields }
func (*Meta) Descriptor
deprecated
func (*Meta) GetImageId ¶
func (*Meta) GetStats ¶
func (x *Meta) GetStats() *pps.ProcessStats
func (*Meta) MarshalLogObject ¶
func (x *Meta) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*Meta) ProtoMessage ¶
func (*Meta) ProtoMessage()
func (*Meta) ProtoReflect ¶ added in v2.7.0
func (x *Meta) ProtoReflect() protoreflect.Message
type Option ¶
type Option func(*Datum)
Option configures a datum.
func WithPrefixIndex ¶
func WithPrefixIndex() Option
WithPrefixIndex prefixes the datum directory name (both locally and in PFS) with its index value.
func WithRecoveryCallback ¶
WithRecoveryCallback sets the recovery callback.
type PFSTask ¶
type PFSTask struct { Input *pps.PFSInput `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` PathRange *pfs.PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"` BaseIndex int64 `protobuf:"varint,3,opt,name=base_index,json=baseIndex,proto3" json:"base_index,omitempty"` AuthToken string `protobuf:"bytes,4,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"` // contains filtered or unexported fields }
func (*PFSTask) Descriptor
deprecated
func (*PFSTask) GetAuthToken ¶
func (*PFSTask) GetBaseIndex ¶
func (*PFSTask) GetPathRange ¶
func (*PFSTask) MarshalLogObject ¶
func (x *PFSTask) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*PFSTask) ProtoMessage ¶
func (*PFSTask) ProtoMessage()
func (*PFSTask) ProtoReflect ¶ added in v2.7.0
func (x *PFSTask) ProtoReflect() protoreflect.Message
type PFSTaskResult ¶
type PFSTaskResult struct { FileSetId string `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"` // contains filtered or unexported fields }
func (*PFSTaskResult) Descriptor
deprecated
func (*PFSTaskResult) Descriptor() ([]byte, []int)
Deprecated: Use PFSTaskResult.ProtoReflect.Descriptor instead.
func (*PFSTaskResult) GetFileSetId ¶
func (x *PFSTaskResult) GetFileSetId() string
func (*PFSTaskResult) MarshalLogObject ¶
func (x *PFSTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*PFSTaskResult) ProtoMessage ¶
func (*PFSTaskResult) ProtoMessage()
func (*PFSTaskResult) ProtoReflect ¶ added in v2.7.0
func (x *PFSTaskResult) ProtoReflect() protoreflect.Message
func (*PFSTaskResult) Reset ¶
func (x *PFSTaskResult) Reset()
func (*PFSTaskResult) String ¶
func (x *PFSTaskResult) String() string
type Set ¶
type Set struct {
// contains filtered or unexported fields
}
Set manages a set of datums.
func (*Set) UploadMeta ¶
UploadMeta uploads the meta file for a datum.
type SetOption ¶
type SetOption func(*Set)
SetOption configures a set.
func WithMetaOutput ¶
func WithMetaOutput(mf client.ModifyFile) SetOption
WithMetaOutput sets the Client for the meta output.
func WithPFSOutput ¶
func WithPFSOutput(mf client.ModifyFile) SetOption
WithPFSOutput sets the Client for the pfs output.
type SetSpec ¶
type SetSpec struct { Number int64 `protobuf:"varint,1,opt,name=number,proto3" json:"number,omitempty"` SizeBytes int64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` // contains filtered or unexported fields }
func (*SetSpec) Descriptor
deprecated
func (*SetSpec) GetSizeBytes ¶
func (*SetSpec) MarshalLogObject ¶
func (x *SetSpec) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*SetSpec) ProtoMessage ¶
func (*SetSpec) ProtoMessage()
func (*SetSpec) ProtoReflect ¶ added in v2.7.0
func (x *SetSpec) ProtoReflect() protoreflect.Message
type State ¶
type State int32
func (State) Descriptor ¶ added in v2.7.0
func (State) Descriptor() protoreflect.EnumDescriptor
func (State) EnumDescriptor
deprecated
func (State) Number ¶ added in v2.7.0
func (x State) Number() protoreflect.EnumNumber
func (State) Type ¶ added in v2.7.0
func (State) Type() protoreflect.EnumType
type Stats ¶
type Stats struct { ProcessStats *pps.ProcessStats `protobuf:"bytes,1,opt,name=process_stats,json=processStats,proto3" json:"process_stats,omitempty"` Processed int64 `protobuf:"varint,2,opt,name=processed,proto3" json:"processed,omitempty"` Skipped int64 `protobuf:"varint,3,opt,name=skipped,proto3" json:"skipped,omitempty"` Total int64 `protobuf:"varint,4,opt,name=total,proto3" json:"total,omitempty"` Failed int64 `protobuf:"varint,5,opt,name=failed,proto3" json:"failed,omitempty"` Recovered int64 `protobuf:"varint,6,opt,name=recovered,proto3" json:"recovered,omitempty"` FailedId string `protobuf:"bytes,7,opt,name=failed_id,json=failedId,proto3" json:"failed_id,omitempty"` // contains filtered or unexported fields }
func (*Stats) Descriptor
deprecated
func (*Stats) GetFailedId ¶ added in v2.7.0
func (*Stats) GetProcessStats ¶
func (x *Stats) GetProcessStats() *pps.ProcessStats
func (*Stats) GetProcessed ¶
func (*Stats) GetRecovered ¶
func (*Stats) GetSkipped ¶
func (*Stats) MarshalLogObject ¶
func (x *Stats) MarshalLogObject(enc zapcore.ObjectEncoder) error
func (*Stats) ProtoMessage ¶
func (*Stats) ProtoMessage()
func (*Stats) ProtoReflect ¶ added in v2.7.0
func (x *Stats) ProtoReflect() protoreflect.Message