datum

package
v2.7.0-alpha.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 22, 2023 License: Apache-2.0 Imports: 42 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrInvalidLengthDatum        = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowDatum          = fmt.Errorf("proto: integer overflow")
	ErrUnexpectedEndOfGroupDatum = fmt.Errorf("proto: unexpected end of group")
)
View Source 
var KeyTask_Type_name = map[int32]string{
	0: "JOIN",
	1: "GROUP",
}
View Source 
var KeyTask_Type_value = map[string]int32{
	"JOIN":  0,
	"GROUP": 1,
}
View Source 
var MergeTask_Type_name = map[int32]string{
	0: "JOIN",
	1: "GROUP",
}
View Source 
var MergeTask_Type_value = map[string]int32{
	"JOIN":  0,
	"GROUP": 1,
}
View Source 
var State_name = map[int32]string{
	0: "PROCESSED",
	1: "FAILED",
	2: "RECOVERED",
}
View Source 
var State_value = map[string]int32{
	"PROCESSED": 0,
	"FAILED":    1,
	"RECOVERED": 2,
}

Functions

func ComposeFileSets 

func ComposeFileSets(pachClient *client.APIClient, taskDoer task.Doer, fileSetIDs []string) (string, error)

func Create 

func Create(pachClient *client.APIClient, taskDoer task.Doer, input *pps.Input) (string, error)

func CreateEmptyFileSet 

func CreateEmptyFileSet(pachClient *client.APIClient) (string, error)

func CreateSets 

func CreateSets(pachClient *client.APIClient, setSpec *SetSpec, fileSetID string, basePathRange *pfs.PathRange) ([]*pfs.PathRange, error)

func IsTask 

func IsTask(input *types.Any) bool

func IsTaskResult 

func IsTaskResult(output *types.Any) bool

func Merge 

func Merge(dits []Iterator, cb func([]*Meta) error) error

Merge merges multiple datum iterators (key is datum ID).

func MergeProcessStats 

func MergeProcessStats(x, y *pps.ProcessStats) error

MergeProcessStats merges two process stats.

func MergeStats 

func MergeStats(x, y *Stats) error

MergeStats merges two stats.

func ProcessTask 

func ProcessTask(pachClient *client.APIClient, input *types.Any) (*types.Any, error)

func TaskResultFileSets 

func TaskResultFileSets(output *types.Any) ([]string, error)

func WithCreateFileSet 

func WithCreateFileSet(pachClient *client.APIClient, name string, cb func(*Set) error) (string, error)

func WithSet 

func WithSet(cacheClient *pfssync.CacheClient, storageRoot string, cb func(*Set) error, opts ...SetOption) (retErr error)

WithSet provides a scoped environment for a datum set.

Types

type ComposeTask 

type ComposeTask struct {
	FileSetIds           []string `protobuf:"bytes,1,rep,name=file_set_ids,json=fileSetIds,proto3" json:"file_set_ids,omitempty"`
	AuthToken            string   `protobuf:"bytes,2,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*ComposeTask) Descriptor 

func (*ComposeTask) Descriptor() ([]byte, []int)

func (*ComposeTask) GetAuthToken 

func (m *ComposeTask) GetAuthToken() string

func (*ComposeTask) GetFileSetIds 

func (m *ComposeTask) GetFileSetIds() []string

func (*ComposeTask) Marshal 

func (m *ComposeTask) Marshal() (dAtA []byte, err error)

func (*ComposeTask) MarshalLogObject 

func (x *ComposeTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ComposeTask) MarshalTo 

func (m *ComposeTask) MarshalTo(dAtA []byte) (int, error)

func (*ComposeTask) MarshalToSizedBuffer 

func (m *ComposeTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ComposeTask) ProtoMessage 

func (*ComposeTask) ProtoMessage()

func (*ComposeTask) Reset 

func (m *ComposeTask) Reset()

func (*ComposeTask) Size 

func (m *ComposeTask) Size() (n int)

func (*ComposeTask) String 

func (m *ComposeTask) String() string

func (*ComposeTask) Unmarshal 

func (m *ComposeTask) Unmarshal(dAtA []byte) error

func (*ComposeTask) XXX_DiscardUnknown 

func (m *ComposeTask) XXX_DiscardUnknown()

func (*ComposeTask) XXX_Marshal 

func (m *ComposeTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ComposeTask) XXX_Merge 

func (m *ComposeTask) XXX_Merge(src proto.Message)

func (*ComposeTask) XXX_Size 

func (m *ComposeTask) XXX_Size() int

func (*ComposeTask) XXX_Unmarshal 

func (m *ComposeTask) XXX_Unmarshal(b []byte) error

type ComposeTaskResult 

type ComposeTaskResult struct {
	FileSetId            string   `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*ComposeTaskResult) Descriptor 

func (*ComposeTaskResult) Descriptor() ([]byte, []int)

func (*ComposeTaskResult) GetFileSetId 

func (m *ComposeTaskResult) GetFileSetId() string

func (*ComposeTaskResult) Marshal 

func (m *ComposeTaskResult) Marshal() (dAtA []byte, err error)

func (*ComposeTaskResult) MarshalLogObject 

func (x *ComposeTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ComposeTaskResult) MarshalTo 

func (m *ComposeTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*ComposeTaskResult) MarshalToSizedBuffer 

func (m *ComposeTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ComposeTaskResult) ProtoMessage 

func (*ComposeTaskResult) ProtoMessage()

func (*ComposeTaskResult) Reset 

func (m *ComposeTaskResult) Reset()

func (*ComposeTaskResult) Size 

func (m *ComposeTaskResult) Size() (n int)

func (*ComposeTaskResult) String 

func (m *ComposeTaskResult) String() string

func (*ComposeTaskResult) Unmarshal 

func (m *ComposeTaskResult) Unmarshal(dAtA []byte) error

func (*ComposeTaskResult) XXX_DiscardUnknown 

func (m *ComposeTaskResult) XXX_DiscardUnknown()

func (*ComposeTaskResult) XXX_Marshal 

func (m *ComposeTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ComposeTaskResult) XXX_Merge 

func (m *ComposeTaskResult) XXX_Merge(src proto.Message)

func (*ComposeTaskResult) XXX_Size 

func (m *ComposeTaskResult) XXX_Size() int

func (*ComposeTaskResult) XXX_Unmarshal 

func (m *ComposeTaskResult) XXX_Unmarshal(b []byte) error

type CrossTask 

type CrossTask struct {
	FileSetIds           []string       `protobuf:"bytes,1,rep,name=file_set_ids,json=fileSetIds,proto3" json:"file_set_ids,omitempty"`
	BaseFileSetIndex     int64          `protobuf:"varint,2,opt,name=base_file_set_index,json=baseFileSetIndex,proto3" json:"base_file_set_index,omitempty"`
	BaseFileSetPathRange *pfs.PathRange `` /* 127-byte string literal not displayed */
	BaseIndex            int64          `protobuf:"varint,4,opt,name=base_index,json=baseIndex,proto3" json:"base_index,omitempty"`
	AuthToken            string         `protobuf:"bytes,5,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"`
	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
	XXX_unrecognized     []byte         `json:"-"`
	XXX_sizecache        int32          `json:"-"`
}

func (*CrossTask) Descriptor 

func (*CrossTask) Descriptor() ([]byte, []int)

func (*CrossTask) GetAuthToken 

func (m *CrossTask) GetAuthToken() string

func (*CrossTask) GetBaseFileSetIndex 

func (m *CrossTask) GetBaseFileSetIndex() int64

func (*CrossTask) GetBaseFileSetPathRange 

func (m *CrossTask) GetBaseFileSetPathRange() *pfs.PathRange

func (*CrossTask) GetBaseIndex 

func (m *CrossTask) GetBaseIndex() int64

func (*CrossTask) GetFileSetIds 

func (m *CrossTask) GetFileSetIds() []string

func (*CrossTask) Marshal 

func (m *CrossTask) Marshal() (dAtA []byte, err error)

func (*CrossTask) MarshalLogObject 

func (x *CrossTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*CrossTask) MarshalTo 

func (m *CrossTask) MarshalTo(dAtA []byte) (int, error)

func (*CrossTask) MarshalToSizedBuffer 

func (m *CrossTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*CrossTask) ProtoMessage 

func (*CrossTask) ProtoMessage()

func (*CrossTask) Reset 

func (m *CrossTask) Reset()

func (*CrossTask) Size 

func (m *CrossTask) Size() (n int)

func (*CrossTask) String 

func (m *CrossTask) String() string

func (*CrossTask) Unmarshal 

func (m *CrossTask) Unmarshal(dAtA []byte) error

func (*CrossTask) XXX_DiscardUnknown 

func (m *CrossTask) XXX_DiscardUnknown()

func (*CrossTask) XXX_Marshal 

func (m *CrossTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*CrossTask) XXX_Merge 

func (m *CrossTask) XXX_Merge(src proto.Message)

func (*CrossTask) XXX_Size 

func (m *CrossTask) XXX_Size() int

func (*CrossTask) XXX_Unmarshal 

func (m *CrossTask) XXX_Unmarshal(b []byte) error

type CrossTaskResult 

type CrossTaskResult struct {
	FileSetId            string   `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*CrossTaskResult) Descriptor 

func (*CrossTaskResult) Descriptor() ([]byte, []int)

func (*CrossTaskResult) GetFileSetId 

func (m *CrossTaskResult) GetFileSetId() string

func (*CrossTaskResult) Marshal 

func (m *CrossTaskResult) Marshal() (dAtA []byte, err error)

func (*CrossTaskResult) MarshalLogObject 

func (x *CrossTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*CrossTaskResult) MarshalTo 

func (m *CrossTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*CrossTaskResult) MarshalToSizedBuffer 

func (m *CrossTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*CrossTaskResult) ProtoMessage 

func (*CrossTaskResult) ProtoMessage()

func (*CrossTaskResult) Reset 

func (m *CrossTaskResult) Reset()

func (*CrossTaskResult) Size 

func (m *CrossTaskResult) Size() (n int)

func (*CrossTaskResult) String 

func (m *CrossTaskResult) String() string

func (*CrossTaskResult) Unmarshal 

func (m *CrossTaskResult) Unmarshal(dAtA []byte) error

func (*CrossTaskResult) XXX_DiscardUnknown 

func (m *CrossTaskResult) XXX_DiscardUnknown()

func (*CrossTaskResult) XXX_Marshal 

func (m *CrossTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*CrossTaskResult) XXX_Merge 

func (m *CrossTaskResult) XXX_Merge(src proto.Message)

func (*CrossTaskResult) XXX_Size 

func (m *CrossTaskResult) XXX_Size() int

func (*CrossTaskResult) XXX_Unmarshal 

func (m *CrossTaskResult) XXX_Unmarshal(b []byte) error

type Datum 

type Datum struct {
	ID string

	IDPrefix string
	// contains filtered or unexported fields
}

Datum manages a datum.

func (*Datum) MetaStorageRoot 

func (d *Datum) MetaStorageRoot() string

MetaStorageRoot returns the meta storage root.

func (*Datum) PFSStorageRoot 

func (d *Datum) PFSStorageRoot() string

PFSStorageRoot returns the pfs storage root.

func (*Datum) Run 

func (d *Datum) Run(ctx context.Context, cb func(ctx context.Context) error) error

Run provides a scoped environment for the processing of a datum.

type Deleter 

type Deleter func(*Meta) error

Deleter deletes a datum.

func NewDeleter 

func NewDeleter(metaFileWalker fileWalkerFunc, metaOutputClient, pfsOutputClient client.ModifyFile) Deleter

NewDeleter creates a new deleter.

type Hasher 

type Hasher interface {
	// Hash computes the datum hash based on the inputs.
	Hash([]*common.Input) string
}

Hasher is the standard interface for a datum hasher.

type Iterator 

type Iterator interface {
	// Iterate iterates over a set of datums.
	Iterate(func(*Meta) error) error
}

Iterator is the standard interface for a datum iterator.

func NewCommitIterator 

func NewCommitIterator(pachClient *client.APIClient, commit *pfs.Commit, pathRange *pfs.PathRange) Iterator

NewCommitIterator creates an iterator for the specified commit and repo.

func NewFileSetIterator 

func NewFileSetIterator(pachClient *client.APIClient, fsID string, pathRange *pfs.PathRange) Iterator

NewFileSetIterator creates a new fileset iterator.

func NewIterator 

func NewIterator(pachClient *client.APIClient, taskDoer task.Doer, input *pps.Input) (Iterator, error)

NewIterator creates a new datum iterator. TODO: Maybe add a renewer parameter to keep file set alive?

func NewJobIterator 

func NewJobIterator(iterator Iterator, job *pps.Job, hasher Hasher) Iterator

NewJobIterator creates a new job iterator.

type KeyTask 

type KeyTask struct {
	FileSetId            string         `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"`
	PathRange            *pfs.PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	Type                 KeyTask_Type   `protobuf:"varint,3,opt,name=type,proto3,enum=datum.KeyTask_Type" json:"type,omitempty"`
	AuthToken            string         `protobuf:"bytes,4,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"`
	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
	XXX_unrecognized     []byte         `json:"-"`
	XXX_sizecache        int32          `json:"-"`
}

func (*KeyTask) Descriptor 

func (*KeyTask) Descriptor() ([]byte, []int)

func (*KeyTask) GetAuthToken 

func (m *KeyTask) GetAuthToken() string

func (*KeyTask) GetFileSetId 

func (m *KeyTask) GetFileSetId() string

func (*KeyTask) GetPathRange 

func (m *KeyTask) GetPathRange() *pfs.PathRange

func (*KeyTask) GetType 

func (m *KeyTask) GetType() KeyTask_Type

func (*KeyTask) Marshal 

func (m *KeyTask) Marshal() (dAtA []byte, err error)

func (*KeyTask) MarshalLogObject 

func (x *KeyTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*KeyTask) MarshalTo 

func (m *KeyTask) MarshalTo(dAtA []byte) (int, error)

func (*KeyTask) MarshalToSizedBuffer 

func (m *KeyTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*KeyTask) ProtoMessage 

func (*KeyTask) ProtoMessage()

func (*KeyTask) Reset 

func (m *KeyTask) Reset()

func (*KeyTask) Size 

func (m *KeyTask) Size() (n int)

func (*KeyTask) String 

func (m *KeyTask) String() string

func (*KeyTask) Unmarshal 

func (m *KeyTask) Unmarshal(dAtA []byte) error

func (*KeyTask) XXX_DiscardUnknown 

func (m *KeyTask) XXX_DiscardUnknown()

func (*KeyTask) XXX_Marshal 

func (m *KeyTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*KeyTask) XXX_Merge 

func (m *KeyTask) XXX_Merge(src proto.Message)

func (*KeyTask) XXX_Size 

func (m *KeyTask) XXX_Size() int

func (*KeyTask) XXX_Unmarshal 

func (m *KeyTask) XXX_Unmarshal(b []byte) error

type KeyTaskResult 

type KeyTaskResult struct {
	FileSetId            string   `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*KeyTaskResult) Descriptor 

func (*KeyTaskResult) Descriptor() ([]byte, []int)

func (*KeyTaskResult) GetFileSetId 

func (m *KeyTaskResult) GetFileSetId() string

func (*KeyTaskResult) Marshal 

func (m *KeyTaskResult) Marshal() (dAtA []byte, err error)

func (*KeyTaskResult) MarshalLogObject 

func (x *KeyTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*KeyTaskResult) MarshalTo 

func (m *KeyTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*KeyTaskResult) MarshalToSizedBuffer 

func (m *KeyTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*KeyTaskResult) ProtoMessage 

func (*KeyTaskResult) ProtoMessage()

func (*KeyTaskResult) Reset 

func (m *KeyTaskResult) Reset()

func (*KeyTaskResult) Size 

func (m *KeyTaskResult) Size() (n int)

func (*KeyTaskResult) String 

func (m *KeyTaskResult) String() string

func (*KeyTaskResult) Unmarshal 

func (m *KeyTaskResult) Unmarshal(dAtA []byte) error

func (*KeyTaskResult) XXX_DiscardUnknown 

func (m *KeyTaskResult) XXX_DiscardUnknown()

func (*KeyTaskResult) XXX_Marshal 

func (m *KeyTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*KeyTaskResult) XXX_Merge 

func (m *KeyTaskResult) XXX_Merge(src proto.Message)

func (*KeyTaskResult) XXX_Size 

func (m *KeyTaskResult) XXX_Size() int

func (*KeyTaskResult) XXX_Unmarshal 

func (m *KeyTaskResult) XXX_Unmarshal(b []byte) error

type KeyTask_Type 

type KeyTask_Type int32
const (
	KeyTask_JOIN  KeyTask_Type = 0
	KeyTask_GROUP KeyTask_Type = 1
)

func (KeyTask_Type) EnumDescriptor 

func (KeyTask_Type) EnumDescriptor() ([]byte, []int)

func (KeyTask_Type) String 

func (x KeyTask_Type) String() string

type MergeTask 

type MergeTask struct {
	FileSetIds           []string       `protobuf:"bytes,1,rep,name=file_set_ids,json=fileSetIds,proto3" json:"file_set_ids,omitempty"`
	PathRange            *pfs.PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	Type                 MergeTask_Type `protobuf:"varint,3,opt,name=type,proto3,enum=datum.MergeTask_Type" json:"type,omitempty"`
	AuthToken            string         `protobuf:"bytes,4,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"`
	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
	XXX_unrecognized     []byte         `json:"-"`
	XXX_sizecache        int32          `json:"-"`
}

func (*MergeTask) Descriptor 

func (*MergeTask) Descriptor() ([]byte, []int)

func (*MergeTask) GetAuthToken 

func (m *MergeTask) GetAuthToken() string

func (*MergeTask) GetFileSetIds 

func (m *MergeTask) GetFileSetIds() []string

func (*MergeTask) GetPathRange 

func (m *MergeTask) GetPathRange() *pfs.PathRange

func (*MergeTask) GetType 

func (m *MergeTask) GetType() MergeTask_Type

func (*MergeTask) Marshal 

func (m *MergeTask) Marshal() (dAtA []byte, err error)

func (*MergeTask) MarshalLogObject 

func (x *MergeTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*MergeTask) MarshalTo 

func (m *MergeTask) MarshalTo(dAtA []byte) (int, error)

func (*MergeTask) MarshalToSizedBuffer 

func (m *MergeTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*MergeTask) ProtoMessage 

func (*MergeTask) ProtoMessage()

func (*MergeTask) Reset 

func (m *MergeTask) Reset()

func (*MergeTask) Size 

func (m *MergeTask) Size() (n int)

func (*MergeTask) String 

func (m *MergeTask) String() string

func (*MergeTask) Unmarshal 

func (m *MergeTask) Unmarshal(dAtA []byte) error

func (*MergeTask) XXX_DiscardUnknown 

func (m *MergeTask) XXX_DiscardUnknown()

func (*MergeTask) XXX_Marshal 

func (m *MergeTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*MergeTask) XXX_Merge 

func (m *MergeTask) XXX_Merge(src proto.Message)

func (*MergeTask) XXX_Size 

func (m *MergeTask) XXX_Size() int

func (*MergeTask) XXX_Unmarshal 

func (m *MergeTask) XXX_Unmarshal(b []byte) error

type MergeTaskResult 

type MergeTaskResult struct {
	FileSetId            string   `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*MergeTaskResult) Descriptor 

func (*MergeTaskResult) Descriptor() ([]byte, []int)

func (*MergeTaskResult) GetFileSetId 

func (m *MergeTaskResult) GetFileSetId() string

func (*MergeTaskResult) Marshal 

func (m *MergeTaskResult) Marshal() (dAtA []byte, err error)

func (*MergeTaskResult) MarshalLogObject 

func (x *MergeTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*MergeTaskResult) MarshalTo 

func (m *MergeTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*MergeTaskResult) MarshalToSizedBuffer 

func (m *MergeTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*MergeTaskResult) ProtoMessage 

func (*MergeTaskResult) ProtoMessage()

func (*MergeTaskResult) Reset 

func (m *MergeTaskResult) Reset()

func (*MergeTaskResult) Size 

func (m *MergeTaskResult) Size() (n int)

func (*MergeTaskResult) String 

func (m *MergeTaskResult) String() string

func (*MergeTaskResult) Unmarshal 

func (m *MergeTaskResult) Unmarshal(dAtA []byte) error

func (*MergeTaskResult) XXX_DiscardUnknown 

func (m *MergeTaskResult) XXX_DiscardUnknown()

func (*MergeTaskResult) XXX_Marshal 

func (m *MergeTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*MergeTaskResult) XXX_Merge 

func (m *MergeTaskResult) XXX_Merge(src proto.Message)

func (*MergeTaskResult) XXX_Size 

func (m *MergeTaskResult) XXX_Size() int

func (*MergeTaskResult) XXX_Unmarshal 

func (m *MergeTaskResult) XXX_Unmarshal(b []byte) error

type MergeTask_Type 

type MergeTask_Type int32
const (
	MergeTask_JOIN  MergeTask_Type = 0
	MergeTask_GROUP MergeTask_Type = 1
)

func (MergeTask_Type) EnumDescriptor 

func (MergeTask_Type) EnumDescriptor() ([]byte, []int)

func (MergeTask_Type) String 

func (x MergeTask_Type) String() string

type Meta 

type Meta struct {
	Job                  *pps.Job          `protobuf:"bytes,1,opt,name=job,proto3" json:"job,omitempty"`
	Inputs               []*common.Input   `protobuf:"bytes,2,rep,name=inputs,proto3" json:"inputs,omitempty"`
	Hash                 string            `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
	State                State             `protobuf:"varint,4,opt,name=state,proto3,enum=datum.State" json:"state,omitempty"`
	Reason               string            `protobuf:"bytes,5,opt,name=reason,proto3" json:"reason,omitempty"`
	Stats                *pps.ProcessStats `protobuf:"bytes,6,opt,name=stats,proto3" json:"stats,omitempty"`
	Index                int64             `protobuf:"varint,7,opt,name=index,proto3" json:"index,omitempty"`
	ImageId              string            `protobuf:"bytes,8,opt,name=image_id,json=imageId,proto3" json:"image_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
	XXX_unrecognized     []byte            `json:"-"`
	XXX_sizecache        int32             `json:"-"`
}

func (*Meta) Descriptor 

func (*Meta) Descriptor() ([]byte, []int)

func (*Meta) GetHash 

func (m *Meta) GetHash() string

func (*Meta) GetImageId 

func (m *Meta) GetImageId() string

func (*Meta) GetIndex 

func (m *Meta) GetIndex() int64

func (*Meta) GetInputs 

func (m *Meta) GetInputs() []*common.Input

func (*Meta) GetJob 

func (m *Meta) GetJob() *pps.Job

func (*Meta) GetReason 

func (m *Meta) GetReason() string

func (*Meta) GetState 

func (m *Meta) GetState() State

func (*Meta) GetStats 

func (m *Meta) GetStats() *pps.ProcessStats

func (*Meta) Marshal 

func (m *Meta) Marshal() (dAtA []byte, err error)

func (*Meta) MarshalLogObject 

func (x *Meta) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*Meta) MarshalTo 

func (m *Meta) MarshalTo(dAtA []byte) (int, error)

func (*Meta) MarshalToSizedBuffer 

func (m *Meta) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Meta) ProtoMessage 

func (*Meta) ProtoMessage()

func (*Meta) Reset 

func (m *Meta) Reset()

func (*Meta) Size 

func (m *Meta) Size() (n int)

func (*Meta) String 

func (m *Meta) String() string

func (*Meta) Unmarshal 

func (m *Meta) Unmarshal(dAtA []byte) error

func (*Meta) XXX_DiscardUnknown 

func (m *Meta) XXX_DiscardUnknown()

func (*Meta) XXX_Marshal 

func (m *Meta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Meta) XXX_Merge 

func (m *Meta) XXX_Merge(src proto.Message)

func (*Meta) XXX_Size 

func (m *Meta) XXX_Size() int

func (*Meta) XXX_Unmarshal 

func (m *Meta) XXX_Unmarshal(b []byte) error

type Option 

type Option func(*Datum)

Option configures a datum.

func WithEnv 

func WithEnv(env []string) Option

WithEnv sets the environment variables.

func WithPrefixIndex 

func WithPrefixIndex() Option

WithPrefixIndex prefixes the datum directory name (both locally and in PFS) with its index value.

func WithRecoveryCallback 

func WithRecoveryCallback(cb func(context.Context) error) Option

WithRecoveryCallback sets the recovery callback.

func WithRetry 

func WithRetry(numRetries int) Option

WithRetry sets the number of retries.

func WithTimeout 

func WithTimeout(timeout time.Duration) Option

WithTimeout sets the timeout.

type PFSTask 

type PFSTask struct {
	Input                *pps.PFSInput  `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"`
	PathRange            *pfs.PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	BaseIndex            int64          `protobuf:"varint,3,opt,name=base_index,json=baseIndex,proto3" json:"base_index,omitempty"`
	AuthToken            string         `protobuf:"bytes,4,opt,name=auth_token,json=authToken,proto3" json:"auth_token,omitempty"`
	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
	XXX_unrecognized     []byte         `json:"-"`
	XXX_sizecache        int32          `json:"-"`
}

func (*PFSTask) Descriptor 

func (*PFSTask) Descriptor() ([]byte, []int)

func (*PFSTask) GetAuthToken 

func (m *PFSTask) GetAuthToken() string

func (*PFSTask) GetBaseIndex 

func (m *PFSTask) GetBaseIndex() int64

func (*PFSTask) GetInput 

func (m *PFSTask) GetInput() *pps.PFSInput

func (*PFSTask) GetPathRange 

func (m *PFSTask) GetPathRange() *pfs.PathRange

func (*PFSTask) Marshal 

func (m *PFSTask) Marshal() (dAtA []byte, err error)

func (*PFSTask) MarshalLogObject 

func (x *PFSTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PFSTask) MarshalTo 

func (m *PFSTask) MarshalTo(dAtA []byte) (int, error)

func (*PFSTask) MarshalToSizedBuffer 

func (m *PFSTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*PFSTask) ProtoMessage 

func (*PFSTask) ProtoMessage()

func (*PFSTask) Reset 

func (m *PFSTask) Reset()

func (*PFSTask) Size 

func (m *PFSTask) Size() (n int)

func (*PFSTask) String 

func (m *PFSTask) String() string

func (*PFSTask) Unmarshal 

func (m *PFSTask) Unmarshal(dAtA []byte) error

func (*PFSTask) XXX_DiscardUnknown 

func (m *PFSTask) XXX_DiscardUnknown()

func (*PFSTask) XXX_Marshal 

func (m *PFSTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*PFSTask) XXX_Merge 

func (m *PFSTask) XXX_Merge(src proto.Message)

func (*PFSTask) XXX_Size 

func (m *PFSTask) XXX_Size() int

func (*PFSTask) XXX_Unmarshal 

func (m *PFSTask) XXX_Unmarshal(b []byte) error

type PFSTaskResult 

type PFSTaskResult struct {
	FileSetId            string   `protobuf:"bytes,1,opt,name=file_set_id,json=fileSetId,proto3" json:"file_set_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*PFSTaskResult) Descriptor 

func (*PFSTaskResult) Descriptor() ([]byte, []int)

func (*PFSTaskResult) GetFileSetId 

func (m *PFSTaskResult) GetFileSetId() string

func (*PFSTaskResult) Marshal 

func (m *PFSTaskResult) Marshal() (dAtA []byte, err error)

func (*PFSTaskResult) MarshalLogObject 

func (x *PFSTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PFSTaskResult) MarshalTo 

func (m *PFSTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*PFSTaskResult) MarshalToSizedBuffer 

func (m *PFSTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*PFSTaskResult) ProtoMessage 

func (*PFSTaskResult) ProtoMessage()

func (*PFSTaskResult) Reset 

func (m *PFSTaskResult) Reset()

func (*PFSTaskResult) Size 

func (m *PFSTaskResult) Size() (n int)

func (*PFSTaskResult) String 

func (m *PFSTaskResult) String() string

func (*PFSTaskResult) Unmarshal 

func (m *PFSTaskResult) Unmarshal(dAtA []byte) error

func (*PFSTaskResult) XXX_DiscardUnknown 

func (m *PFSTaskResult) XXX_DiscardUnknown()

func (*PFSTaskResult) XXX_Marshal 

func (m *PFSTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*PFSTaskResult) XXX_Merge 

func (m *PFSTaskResult) XXX_Merge(src proto.Message)

func (*PFSTaskResult) XXX_Size 

func (m *PFSTaskResult) XXX_Size() int

func (*PFSTaskResult) XXX_Unmarshal 

func (m *PFSTaskResult) XXX_Unmarshal(b []byte) error

type Set 

type Set struct {
	// contains filtered or unexported fields
}

Set manages a set of datums.

func (*Set) UploadMeta 

func (s *Set) UploadMeta(meta *Meta, opts ...Option) error

UploadMeta uploads the meta file for a datum.

func (*Set) WithDatum 

func (s *Set) WithDatum(meta *Meta, cb func(*Datum) error, opts ...Option) error

WithDatum provides a scoped environment for a datum within the datum set. TODO: Handle datum concurrency here, and potentially move symlinking here.

type SetOption 

type SetOption func(*Set)

SetOption configures a set.

func WithMetaOutput 

func WithMetaOutput(mf client.ModifyFile) SetOption

WithMetaOutput sets the Client for the meta output.

func WithPFSOutput 

func WithPFSOutput(mf client.ModifyFile) SetOption

WithPFSOutput sets the Client for the pfs output.

func WithStats 

func WithStats(stats *Stats) SetOption

WithStats sets the stats to fill in.

type SetSpec 

type SetSpec struct {
	Number               int64    `protobuf:"varint,1,opt,name=number,proto3" json:"number,omitempty"`
	SizeBytes            int64    `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*SetSpec) Descriptor 

func (*SetSpec) Descriptor() ([]byte, []int)

func (*SetSpec) GetNumber 

func (m *SetSpec) GetNumber() int64

func (*SetSpec) GetSizeBytes 

func (m *SetSpec) GetSizeBytes() int64

func (*SetSpec) Marshal 

func (m *SetSpec) Marshal() (dAtA []byte, err error)

func (*SetSpec) MarshalLogObject 

func (x *SetSpec) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*SetSpec) MarshalTo 

func (m *SetSpec) MarshalTo(dAtA []byte) (int, error)

func (*SetSpec) MarshalToSizedBuffer 

func (m *SetSpec) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*SetSpec) ProtoMessage 

func (*SetSpec) ProtoMessage()

func (*SetSpec) Reset 

func (m *SetSpec) Reset()

func (*SetSpec) Size 

func (m *SetSpec) Size() (n int)

func (*SetSpec) String 

func (m *SetSpec) String() string

func (*SetSpec) Unmarshal 

func (m *SetSpec) Unmarshal(dAtA []byte) error

func (*SetSpec) XXX_DiscardUnknown 

func (m *SetSpec) XXX_DiscardUnknown()

func (*SetSpec) XXX_Marshal 

func (m *SetSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*SetSpec) XXX_Merge 

func (m *SetSpec) XXX_Merge(src proto.Message)

func (*SetSpec) XXX_Size 

func (m *SetSpec) XXX_Size() int

func (*SetSpec) XXX_Unmarshal 

func (m *SetSpec) XXX_Unmarshal(b []byte) error

type State 

type State int32
const (
	State_PROCESSED State = 0
	State_FAILED    State = 1
	State_RECOVERED State = 2
)

func (State) EnumDescriptor 

func (State) EnumDescriptor() ([]byte, []int)

func (State) String 

func (x State) String() string

type Stats 

type Stats struct {
	ProcessStats         *pps.ProcessStats `protobuf:"bytes,1,opt,name=process_stats,json=processStats,proto3" json:"process_stats,omitempty"`
	Processed            int64             `protobuf:"varint,2,opt,name=processed,proto3" json:"processed,omitempty"`
	Skipped              int64             `protobuf:"varint,3,opt,name=skipped,proto3" json:"skipped,omitempty"`
	Total                int64             `protobuf:"varint,4,opt,name=total,proto3" json:"total,omitempty"`
	Failed               int64             `protobuf:"varint,5,opt,name=failed,proto3" json:"failed,omitempty"`
	Recovered            int64             `protobuf:"varint,6,opt,name=recovered,proto3" json:"recovered,omitempty"`
	FailedID             string            `protobuf:"bytes,7,opt,name=failed_id,json=failedId,proto3" json:"failed_id,omitempty"`
	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
	XXX_unrecognized     []byte            `json:"-"`
	XXX_sizecache        int32             `json:"-"`
}

func (*Stats) Descriptor 

func (*Stats) Descriptor() ([]byte, []int)

func (*Stats) GetFailed 

func (m *Stats) GetFailed() int64

func (*Stats) GetFailedID 

func (m *Stats) GetFailedID() string

func (*Stats) GetProcessStats 

func (m *Stats) GetProcessStats() *pps.ProcessStats

func (*Stats) GetProcessed 

func (m *Stats) GetProcessed() int64

func (*Stats) GetRecovered 

func (m *Stats) GetRecovered() int64

func (*Stats) GetSkipped 

func (m *Stats) GetSkipped() int64

func (*Stats) GetTotal 

func (m *Stats) GetTotal() int64

func (*Stats) Marshal 

func (m *Stats) Marshal() (dAtA []byte, err error)

func (*Stats) MarshalLogObject 

func (x *Stats) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*Stats) MarshalTo 

func (m *Stats) MarshalTo(dAtA []byte) (int, error)

func (*Stats) MarshalToSizedBuffer 

func (m *Stats) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Stats) ProtoMessage 

func (*Stats) ProtoMessage()

func (*Stats) Reset 

func (m *Stats) Reset()

func (*Stats) Size 

func (m *Stats) Size() (n int)

func (*Stats) String 

func (m *Stats) String() string

func (*Stats) Unmarshal 

func (m *Stats) Unmarshal(dAtA []byte) error

func (*Stats) XXX_DiscardUnknown 

func (m *Stats) XXX_DiscardUnknown()

func (*Stats) XXX_Marshal 

func (m *Stats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Stats) XXX_Merge 

func (m *Stats) XXX_Merge(src proto.Message)

func (*Stats) XXX_Size 

func (m *Stats) XXX_Size() int

func (*Stats) XXX_Unmarshal 

func (m *Stats) XXX_Unmarshal(b []byte) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.