common

Documentation

Index

• Constants
• Variables
• func DatumID(inputs []*Input) string
• func HashDatum(pipelineSalt string, inputs []*Input) string
• func MatchDatum(filter []string, inputs []*pps.InputFile) bool
• func MetaFilePath(id string) string
• func Shard(pachClient *client.APIClient, fileSetIDs []string) ([]*pfs.PathRange, error)
• type Input
  • func (*Input) Descriptor() ([]byte, []int)
  • func (x *Input) GetBranch() string
  • func (x *Input) GetEmptyFiles() bool
  • func (x *Input) GetFileInfo() *pfs.FileInfo
  • func (x *Input) GetGroupBy() string
  • func (x *Input) GetJoinOn() string
  • func (x *Input) GetLazy() bool
  • func (x *Input) GetName() string
  • func (x *Input) GetOuterJoin() bool
  • func (x *Input) GetS3() bool
  • func (x *Input) MarshalLogObject(enc zapcore.ObjectEncoder) error
  • func (*Input) ProtoMessage()
  • func (x *Input) ProtoReflect() protoreflect.Message
  • func (x *Input) Reset()
  • func (x *Input) String() string

Constants

const (
	// MetaPrefix is the prefix for the meta path.
	MetaPrefix = "meta"
	// MetaFileName is the name of the meta file.
	MetaFileName = "meta"
	// PFSPrefix is the prefix for the pfs path.
	PFSPrefix = "pfs"
	// OutputPrefix is the prefix for the output path.
	OutputPrefix = "out"
	// TmpFileName is the name of the tmp file.
	TmpFileName = "tmp"
	// EnvFileName is the name of the env file.
	EnvFileName = ".env"
	TTL         = 15 * time.Minute
)

Variables

var File_server_worker_common_common_proto protoreflect.FileDescriptor

Functions

func DatumID

func DatumID(inputs []*Input) string

DatumID computes the ID of a datum.

func HashDatum

func HashDatum(pipelineSalt string, inputs []*Input) string

HashDatum computes the hash of a datum.

func MatchDatum

func MatchDatum(filter []string, inputs []*pps.InputFile) bool

MatchDatum checks if a datum matches a filter. To match each string in filter must correspond match at least 1 datum's Path or Hash. Order of filter and inputs is irrelevant.

func MetaFilePath

func MetaFilePath(id string) string

func Shard

func Shard(pachClient *client.APIClient, fileSetIDs []string) ([]*pfs.PathRange, error)

TODO: Trim non-meta file shards?

Types

type Input

type Input struct {
	FileInfo   *pfs.FileInfo `protobuf:"bytes,1,opt,name=file_info,json=fileInfo,proto3" json:"file_info,omitempty"`
	Name       string        `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
	JoinOn     string        `protobuf:"bytes,4,opt,name=join_on,json=joinOn,proto3" json:"join_on,omitempty"`
	OuterJoin  bool          `protobuf:"varint,5,opt,name=outer_join,json=outerJoin,proto3" json:"outer_join,omitempty"`
	GroupBy    string        `protobuf:"bytes,6,opt,name=group_by,json=groupBy,proto3" json:"group_by,omitempty"`
	Lazy       bool          `protobuf:"varint,7,opt,name=lazy,proto3" json:"lazy,omitempty"`
	Branch     string        `protobuf:"bytes,8,opt,name=branch,proto3" json:"branch,omitempty"`
	EmptyFiles bool          `protobuf:"varint,10,opt,name=empty_files,json=emptyFiles,proto3" json:"empty_files,omitempty"`
	S3         bool          `protobuf:"varint,11,opt,name=s3,proto3" json:"s3,omitempty"` // If set, workers won't create an input directory for this input
	// contains filtered or unexported fields
}

func (*Input) Descriptor

func (*Input) Descriptor() ([]byte, []int)

Deprecated: Use Input.ProtoReflect.Descriptor instead.

func (*Input) GetBranch

func (x *Input) GetBranch() string

func (*Input) GetEmptyFiles

func (x *Input) GetEmptyFiles() bool

func (*Input) GetFileInfo

func (x *Input) GetFileInfo() *pfs.FileInfo

func (*Input) GetGroupBy

func (x *Input) GetGroupBy() string

func (*Input) GetJoinOn

func (x *Input) GetJoinOn() string

func (*Input) GetLazy

func (x *Input) GetLazy() bool

func (*Input) GetName

func (x *Input) GetName() string

func (*Input) GetOuterJoin

func (x *Input) GetOuterJoin() bool

func (*Input) GetS3

func (x *Input) GetS3() bool

func (*Input) MarshalLogObject

func (x *Input) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*Input) ProtoMessage

func (*Input) ProtoMessage()

func (*Input) ProtoReflect

func (x *Input) ProtoReflect() protoreflect.Message

func (*Input) Reset

func (x *Input) Reset()

func (*Input) String

func (x *Input) String() string