common

Documentation

Index

• Constants
• Variables
• func DatumID(inputs []*Input) string
• func HashDatum(pipelineSalt string, inputs []*Input) string
• func MatchDatum(filter []string, inputs []*pps.InputFile) bool
• func MetaFilePath(id string) string
• func Shard(pachClient *client.APIClient, fileSetIDs []string) ([]*pfs.PathRange, error)
• type Input
  • func (*Input) Descriptor() ([]byte, []int)
  • func (m *Input) GetBranch() string
  • func (m *Input) GetEmptyFiles() bool
  • func (m *Input) GetFileInfo() *pfs.FileInfo
  • func (m *Input) GetGitURL() string
  • func (m *Input) GetGroupBy() string
  • func (m *Input) GetJoinOn() string
  • func (m *Input) GetLazy() bool
  • func (m *Input) GetName() string
  • func (m *Input) GetOuterJoin() bool
  • func (m *Input) GetS3() bool
  • func (m *Input) Marshal() (dAtA []byte, err error)
  • func (x *Input) MarshalLogObject(enc zapcore.ObjectEncoder) error
  • func (m *Input) MarshalTo(dAtA []byte) (int, error)
  • func (m *Input) MarshalToSizedBuffer(dAtA []byte) (int, error)
  • func (*Input) ProtoMessage()
  • func (m *Input) Reset()
  • func (m *Input) Size() (n int)
  • func (m *Input) String() string
  • func (m *Input) Unmarshal(dAtA []byte) error
  • func (m *Input) XXX_DiscardUnknown()
  • func (m *Input) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
  • func (m *Input) XXX_Merge(src proto.Message)
  • func (m *Input) XXX_Size() int
  • func (m *Input) XXX_Unmarshal(b []byte) error

Constants

const (
	// MetaPrefix is the prefix for the meta path.
	MetaPrefix = "meta"
	// MetaFileName is the name of the meta file.
	MetaFileName = "meta"
	// PFSPrefix is the prefix for the pfs path.
	PFSPrefix = "pfs"
	// OutputPrefix is the prefix for the output path.
	OutputPrefix = "out"
	// TmpFileName is the name of the tmp file.
	TmpFileName = "tmp"
	// EnvFileName is the name of the env file.
	EnvFileName = ".env"
	TTL         = 15 * time.Minute
)

Variables

var (
	ErrInvalidLengthCommon        = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowCommon          = fmt.Errorf("proto: integer overflow")
	ErrUnexpectedEndOfGroupCommon = fmt.Errorf("proto: unexpected end of group")
)

Functions

func DatumID

func DatumID(inputs []*Input) string

DatumID computes the ID of a datum.

func HashDatum

func HashDatum(pipelineSalt string, inputs []*Input) string

HashDatum computes the hash of a datum.

func MatchDatum

func MatchDatum(filter []string, inputs []*pps.InputFile) bool

MatchDatum checks if a datum matches a filter. To match each string in filter must correspond match at least 1 datum's Path or Hash. Order of filter and inputs is irrelevant.

func MetaFilePath

func MetaFilePath(id string) string

func Shard

func Shard(pachClient *client.APIClient, fileSetIDs []string) ([]*pfs.PathRange, error)

TODO: Trim non-meta file shards?

Types

type Input

type Input struct {
	FileInfo             *pfs.FileInfo `protobuf:"bytes,1,opt,name=file_info,json=fileInfo,proto3" json:"file_info,omitempty"`
	Name                 string        `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"`
	JoinOn               string        `protobuf:"bytes,4,opt,name=join_on,json=joinOn,proto3" json:"join_on,omitempty"`
	OuterJoin            bool          `protobuf:"varint,5,opt,name=outer_join,json=outerJoin,proto3" json:"outer_join,omitempty"`
	GroupBy              string        `protobuf:"bytes,6,opt,name=group_by,json=groupBy,proto3" json:"group_by,omitempty"`
	Lazy                 bool          `protobuf:"varint,7,opt,name=lazy,proto3" json:"lazy,omitempty"`
	Branch               string        `protobuf:"bytes,8,opt,name=branch,proto3" json:"branch,omitempty"`
	GitURL               string        `protobuf:"bytes,9,opt,name=git_url,json=gitUrl,proto3" json:"git_url,omitempty"`
	EmptyFiles           bool          `protobuf:"varint,10,opt,name=empty_files,json=emptyFiles,proto3" json:"empty_files,omitempty"`
	S3                   bool          `protobuf:"varint,11,opt,name=s3,proto3" json:"s3,omitempty"`
	XXX_NoUnkeyedLiteral struct{}      `json:"-"`
	XXX_unrecognized     []byte        `json:"-"`
	XXX_sizecache        int32         `json:"-"`
}

func (*Input) Descriptor

func (*Input) Descriptor() ([]byte, []int)

func (*Input) GetBranch

func (m *Input) GetBranch() string

func (*Input) GetEmptyFiles

func (m *Input) GetEmptyFiles() bool

func (*Input) GetFileInfo

func (m *Input) GetFileInfo() *pfs.FileInfo

func (*Input) GetGitURL

func (m *Input) GetGitURL() string

func (*Input) GetGroupBy

func (m *Input) GetGroupBy() string

func (*Input) GetJoinOn

func (m *Input) GetJoinOn() string

func (*Input) GetLazy

func (m *Input) GetLazy() bool

func (*Input) GetName

func (m *Input) GetName() string

func (*Input) GetOuterJoin

func (m *Input) GetOuterJoin() bool

func (*Input) GetS3

func (m *Input) GetS3() bool

func (*Input) Marshal

func (m *Input) Marshal() (dAtA []byte, err error)

func (*Input) MarshalLogObject

func (x *Input) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*Input) MarshalTo

func (m *Input) MarshalTo(dAtA []byte) (int, error)

func (*Input) MarshalToSizedBuffer

func (m *Input) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Input) ProtoMessage

func (*Input) ProtoMessage()

func (*Input) Reset

func (m *Input) Reset()

func (*Input) Size

func (m *Input) Size() (n int)

func (*Input) String

func (m *Input) String() string

func (*Input) Unmarshal

func (m *Input) Unmarshal(dAtA []byte) error

func (*Input) XXX_DiscardUnknown

func (m *Input) XXX_DiscardUnknown()

func (*Input) XXX_Marshal

func (m *Input) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Input) XXX_Merge

func (m *Input) XXX_Merge(src proto.Message)

func (*Input) XXX_Size

func (m *Input) XXX_Size() int

func (*Input) XXX_Unmarshal

func (m *Input) XXX_Unmarshal(b []byte) error