server

package
v2.7.0-nightly.20230626 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 23, 2023 License: Apache-2.0 Imports: 86 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	StorageTaskNamespace = "storage"
)
View Source 
const (
	URLTaskNamespace = "url"
)

Variables

View Source 
var (
	ErrInvalidLengthPfsserver        = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowPfsserver          = fmt.Errorf("proto: integer overflow")
	ErrUnexpectedEndOfGroupPfsserver = fmt.Errorf("proto: unexpected end of group")
)

Functions

func IsPermissionError 

func IsPermissionError(err error) bool

IsPermissionError returns true if a given error is a permission error.

func NewAPIServer 

func NewAPIServer(env Env) (pfsserver.APIServer, error)

NewAPIServer creates an APIServer.

func NewPachwAPIServer 

func NewPachwAPIServer(env Env) (pfsserver.APIServer, error)

NewPachwAPIServer is used when running pachd in Pachw Mode. In Pachw Mode, a pachd instance processes storage and URl related tasks via the task service.

func NewSidecarAPIServer 

func NewSidecarAPIServer(env Env) (pfsserver.APIServer, error)

func SetupPostgresCommitStoreV0 

func SetupPostgresCommitStoreV0(ctx context.Context, tx *pachsql.Tx) error

SetupPostgresCommitStoreV0 runs SQL to setup the commit store. DO NOT MODIFY THIS FUNCTION IT HAS BEEN USED IN A RELEASED MIGRATION

func TopologicalSort 

func TopologicalSort(cis []*pfs.CommitInfo) []*pfs.CommitInfo

TopologicalSort sorts a slice of commit infos topologically based on their provenance

func ValidateFilename 

func ValidateFilename(p string) error

Types

type CommitEvent 

type CommitEvent struct {
	Err   error
	Value *pfs.CommitInfo
}

CommitEvent is an event that contains a CommitInfo or an error

type CommitStream 

type CommitStream interface {
	Stream() <-chan CommitEvent
	Close()
}

CommitStream is a stream of CommitInfos

type CompactTask 

type CompactTask struct {
	Inputs               []string   `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
	PathRange            *PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
	XXX_unrecognized     []byte     `json:"-"`
	XXX_sizecache        int32      `json:"-"`
}

func (*CompactTask) Descriptor 

func (*CompactTask) Descriptor() ([]byte, []int)

func (*CompactTask) GetInputs 

func (m *CompactTask) GetInputs() []string

func (*CompactTask) GetPathRange 

func (m *CompactTask) GetPathRange() *PathRange

func (*CompactTask) Marshal 

func (m *CompactTask) Marshal() (dAtA []byte, err error)

func (*CompactTask) MarshalLogObject 

func (x *CompactTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*CompactTask) MarshalTo 

func (m *CompactTask) MarshalTo(dAtA []byte) (int, error)

func (*CompactTask) MarshalToSizedBuffer 

func (m *CompactTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*CompactTask) ProtoMessage 

func (*CompactTask) ProtoMessage()

func (*CompactTask) Reset 

func (m *CompactTask) Reset()

func (*CompactTask) Size 

func (m *CompactTask) Size() (n int)

func (*CompactTask) String 

func (m *CompactTask) String() string

func (*CompactTask) Unmarshal 

func (m *CompactTask) Unmarshal(dAtA []byte) error

func (*CompactTask) XXX_DiscardUnknown 

func (m *CompactTask) XXX_DiscardUnknown()

func (*CompactTask) XXX_Marshal 

func (m *CompactTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*CompactTask) XXX_Merge 

func (m *CompactTask) XXX_Merge(src proto.Message)

func (*CompactTask) XXX_Size 

func (m *CompactTask) XXX_Size() int

func (*CompactTask) XXX_Unmarshal 

func (m *CompactTask) XXX_Unmarshal(b []byte) error

type CompactTaskResult 

type CompactTaskResult struct {
	Id                   string   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*CompactTaskResult) Descriptor 

func (*CompactTaskResult) Descriptor() ([]byte, []int)

func (*CompactTaskResult) GetId 

func (m *CompactTaskResult) GetId() string

func (*CompactTaskResult) Marshal 

func (m *CompactTaskResult) Marshal() (dAtA []byte, err error)

func (*CompactTaskResult) MarshalLogObject 

func (x *CompactTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*CompactTaskResult) MarshalTo 

func (m *CompactTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*CompactTaskResult) MarshalToSizedBuffer 

func (m *CompactTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*CompactTaskResult) ProtoMessage 

func (*CompactTaskResult) ProtoMessage()

func (*CompactTaskResult) Reset 

func (m *CompactTaskResult) Reset()

func (*CompactTaskResult) Size 

func (m *CompactTaskResult) Size() (n int)

func (*CompactTaskResult) String 

func (m *CompactTaskResult) String() string

func (*CompactTaskResult) Unmarshal 

func (m *CompactTaskResult) Unmarshal(dAtA []byte) error

func (*CompactTaskResult) XXX_DiscardUnknown 

func (m *CompactTaskResult) XXX_DiscardUnknown()

func (*CompactTaskResult) XXX_Marshal 

func (m *CompactTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*CompactTaskResult) XXX_Merge 

func (m *CompactTaskResult) XXX_Merge(src proto.Message)

func (*CompactTaskResult) XXX_Size 

func (m *CompactTaskResult) XXX_Size() int

func (*CompactTaskResult) XXX_Unmarshal 

func (m *CompactTaskResult) XXX_Unmarshal(b []byte) error

type ConcatTask 

type ConcatTask struct {
	Inputs               []string `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*ConcatTask) Descriptor 

func (*ConcatTask) Descriptor() ([]byte, []int)

func (*ConcatTask) GetInputs 

func (m *ConcatTask) GetInputs() []string

func (*ConcatTask) Marshal 

func (m *ConcatTask) Marshal() (dAtA []byte, err error)

func (*ConcatTask) MarshalLogObject 

func (x *ConcatTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ConcatTask) MarshalTo 

func (m *ConcatTask) MarshalTo(dAtA []byte) (int, error)

func (*ConcatTask) MarshalToSizedBuffer 

func (m *ConcatTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ConcatTask) ProtoMessage 

func (*ConcatTask) ProtoMessage()

func (*ConcatTask) Reset 

func (m *ConcatTask) Reset()

func (*ConcatTask) Size 

func (m *ConcatTask) Size() (n int)

func (*ConcatTask) String 

func (m *ConcatTask) String() string

func (*ConcatTask) Unmarshal 

func (m *ConcatTask) Unmarshal(dAtA []byte) error

func (*ConcatTask) XXX_DiscardUnknown 

func (m *ConcatTask) XXX_DiscardUnknown()

func (*ConcatTask) XXX_Marshal 

func (m *ConcatTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ConcatTask) XXX_Merge 

func (m *ConcatTask) XXX_Merge(src proto.Message)

func (*ConcatTask) XXX_Size 

func (m *ConcatTask) XXX_Size() int

func (*ConcatTask) XXX_Unmarshal 

func (m *ConcatTask) XXX_Unmarshal(b []byte) error

type ConcatTaskResult 

type ConcatTaskResult struct {
	Id                   string   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*ConcatTaskResult) Descriptor 

func (*ConcatTaskResult) Descriptor() ([]byte, []int)

func (*ConcatTaskResult) GetId 

func (m *ConcatTaskResult) GetId() string

func (*ConcatTaskResult) Marshal 

func (m *ConcatTaskResult) Marshal() (dAtA []byte, err error)

func (*ConcatTaskResult) MarshalLogObject 

func (x *ConcatTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ConcatTaskResult) MarshalTo 

func (m *ConcatTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*ConcatTaskResult) MarshalToSizedBuffer 

func (m *ConcatTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ConcatTaskResult) ProtoMessage 

func (*ConcatTaskResult) ProtoMessage()

func (*ConcatTaskResult) Reset 

func (m *ConcatTaskResult) Reset()

func (*ConcatTaskResult) Size 

func (m *ConcatTaskResult) Size() (n int)

func (*ConcatTaskResult) String 

func (m *ConcatTaskResult) String() string

func (*ConcatTaskResult) Unmarshal 

func (m *ConcatTaskResult) Unmarshal(dAtA []byte) error

func (*ConcatTaskResult) XXX_DiscardUnknown 

func (m *ConcatTaskResult) XXX_DiscardUnknown()

func (*ConcatTaskResult) XXX_Marshal 

func (m *ConcatTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ConcatTaskResult) XXX_Merge 

func (m *ConcatTaskResult) XXX_Merge(src proto.Message)

func (*ConcatTaskResult) XXX_Size 

func (m *ConcatTaskResult) XXX_Size() int

func (*ConcatTaskResult) XXX_Unmarshal 

func (m *ConcatTaskResult) XXX_Unmarshal(b []byte) error

type Differ 

type Differ struct {
	// contains filtered or unexported fields
}

Differ compares two sources and iterates over the items that are not equal.

func NewDiffer 

func NewDiffer(a, b Source) *Differ

NewDiffer creates an iterator over the differences between Sources a and b

func (*Differ) Iterate 

func (d *Differ) Iterate(ctx context.Context, cb func(aFi, bFi *pfs.FileInfo) error) error

Iterate compares the entries from `a` and `b` path wise. If one side is missing a path, cb is called with the info for the side that has the path If both sides have a path, but the content is different, cb is called with the info for both sides at once. If both sides have a path, and the content is the same, cb is not called. The info is not part of the diff.

type Env 

type Env struct {
	ObjectClient obj.Client
	DB           *pachsql.DB
	EtcdPrefix   string
	EtcdClient   *etcd.Client
	TaskService  task.Service
	TxnEnv       *txnenv.TransactionEnv
	Listener     col.PostgresListener

	AuthServer authserver.APIServer
	// TODO: a reasonable repo metadata solution would let us get rid of this circular dependency
	// permissions might also work.
	GetPPSServer func() ppsserver.APIServer
	// TODO: remove this, the load tests need a pachClient
	GetPachClient func(ctx context.Context) *client.APIClient

	BackgroundContext context.Context
	StorageConfig     pachconfig.StorageConfiguration
	PachwInSidecar    bool
}

Env is the dependencies needed to run the PFS API server

func EnvFromServiceEnv 

func EnvFromServiceEnv(env serviceenv.ServiceEnv, txnEnv *txnenv.TransactionEnv) (*Env, error)

type ErrBranchCommitProvenanceMismatch 

type ErrBranchCommitProvenanceMismatch struct {
	Branch       *pfs.Branch
	ParentBranch *pfs.Branch
}

ErrBranchCommitProvenanceMismatch occurs when the head commit of one of the parents of the branch is not found in the direct provenance commits of the head of the branch

func (ErrBranchCommitProvenanceMismatch) Error 

func (e ErrBranchCommitProvenanceMismatch) Error() string

type ErrBranchInfoNotFound 

type ErrBranchInfoNotFound struct {
	Branch *pfs.Branch
}

ErrBranchInfoNotFound Branch info could not be found. Typically because of an incomplete deletion of a branch. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrBranchInfoNotFound) Error 

func (e ErrBranchInfoNotFound) Error() string

type ErrBranchProvenanceTransitivity 

type ErrBranchProvenanceTransitivity struct {
	BranchInfo     *pfs.BranchInfo
	FullProvenance []*pfs.Branch
}

ErrBranchProvenanceTransitivity Branch provenance is not transitively closed. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrBranchProvenanceTransitivity) Error 

func (e ErrBranchProvenanceTransitivity) Error() string

type ErrBranchSubvenanceTransitivity 

type ErrBranchSubvenanceTransitivity struct {
	BranchInfo        *pfs.BranchInfo
	MissingSubvenance *pfs.Branch
}

func (ErrBranchSubvenanceTransitivity) Error 

func (e ErrBranchSubvenanceTransitivity) Error() string

type ErrCommitAncestryBroken 

type ErrCommitAncestryBroken struct {
	Parent *pfs.Commit
	Child  *pfs.Commit
}

ErrCommitAncestryBroken indicates that a parent and child commit disagree on their relationship. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrCommitAncestryBroken) Error 

func (e ErrCommitAncestryBroken) Error() string

type ErrCommitInfoNotFound 

type ErrCommitInfoNotFound struct {
	Location string
	Commit   *pfs.Commit
}

ErrCommitInfoNotFound Commit info could not be found. Typically because of an incomplete deletion of a commit. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrCommitInfoNotFound) Error 

func (e ErrCommitInfoNotFound) Error() string

type ErrZombieData 

type ErrZombieData struct {
	Commit *pfs.Commit
	ID     string
}

func (ErrZombieData) Error 

func (e ErrZombieData) Error() string

type GetFileURLTask 

type GetFileURLTask struct {
	URL                  string         `protobuf:"bytes,1,opt,name=URL,proto3" json:"URL,omitempty"`
	File                 *pfs.File      `protobuf:"bytes,2,opt,name=file,proto3" json:"file,omitempty"`
	PathRange            *pfs.PathRange `protobuf:"bytes,3,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
	XXX_unrecognized     []byte         `json:"-"`
	XXX_sizecache        int32          `json:"-"`
}

func (*GetFileURLTask) Descriptor 

func (*GetFileURLTask) Descriptor() ([]byte, []int)

func (*GetFileURLTask) GetFile 

func (m *GetFileURLTask) GetFile() *pfs.File

func (*GetFileURLTask) GetPathRange 

func (m *GetFileURLTask) GetPathRange() *pfs.PathRange

func (*GetFileURLTask) GetURL 

func (m *GetFileURLTask) GetURL() string

func (*GetFileURLTask) Marshal 

func (m *GetFileURLTask) Marshal() (dAtA []byte, err error)

func (*GetFileURLTask) MarshalLogObject 

func (x *GetFileURLTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*GetFileURLTask) MarshalTo 

func (m *GetFileURLTask) MarshalTo(dAtA []byte) (int, error)

func (*GetFileURLTask) MarshalToSizedBuffer 

func (m *GetFileURLTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*GetFileURLTask) ProtoMessage 

func (*GetFileURLTask) ProtoMessage()

func (*GetFileURLTask) Reset 

func (m *GetFileURLTask) Reset()

func (*GetFileURLTask) Size 

func (m *GetFileURLTask) Size() (n int)

func (*GetFileURLTask) String 

func (m *GetFileURLTask) String() string

func (*GetFileURLTask) Unmarshal 

func (m *GetFileURLTask) Unmarshal(dAtA []byte) error

func (*GetFileURLTask) XXX_DiscardUnknown 

func (m *GetFileURLTask) XXX_DiscardUnknown()

func (*GetFileURLTask) XXX_Marshal 

func (m *GetFileURLTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*GetFileURLTask) XXX_Merge 

func (m *GetFileURLTask) XXX_Merge(src proto.Message)

func (*GetFileURLTask) XXX_Size 

func (m *GetFileURLTask) XXX_Size() int

func (*GetFileURLTask) XXX_Unmarshal 

func (m *GetFileURLTask) XXX_Unmarshal(b []byte) error

type GetFileURLTaskResult 

type GetFileURLTaskResult struct {
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*GetFileURLTaskResult) Descriptor 

func (*GetFileURLTaskResult) Descriptor() ([]byte, []int)

func (*GetFileURLTaskResult) Marshal 

func (m *GetFileURLTaskResult) Marshal() (dAtA []byte, err error)

func (*GetFileURLTaskResult) MarshalLogObject 

func (x *GetFileURLTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*GetFileURLTaskResult) MarshalTo 

func (m *GetFileURLTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*GetFileURLTaskResult) MarshalToSizedBuffer 

func (m *GetFileURLTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*GetFileURLTaskResult) ProtoMessage 

func (*GetFileURLTaskResult) ProtoMessage()

func (*GetFileURLTaskResult) Reset 

func (m *GetFileURLTaskResult) Reset()

func (*GetFileURLTaskResult) Size 

func (m *GetFileURLTaskResult) Size() (n int)

func (*GetFileURLTaskResult) String 

func (m *GetFileURLTaskResult) String() string

func (*GetFileURLTaskResult) Unmarshal 

func (m *GetFileURLTaskResult) Unmarshal(dAtA []byte) error

func (*GetFileURLTaskResult) XXX_DiscardUnknown 

func (m *GetFileURLTaskResult) XXX_DiscardUnknown()

func (*GetFileURLTaskResult) XXX_Marshal 

func (m *GetFileURLTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*GetFileURLTaskResult) XXX_Merge 

func (m *GetFileURLTaskResult) XXX_Merge(src proto.Message)

func (*GetFileURLTaskResult) XXX_Size 

func (m *GetFileURLTaskResult) XXX_Size() int

func (*GetFileURLTaskResult) XXX_Unmarshal 

func (m *GetFileURLTaskResult) XXX_Unmarshal(b []byte) error

type PathRange 

type PathRange struct {
	Lower                string   `protobuf:"bytes,1,opt,name=lower,proto3" json:"lower,omitempty"`
	Upper                string   `protobuf:"bytes,2,opt,name=upper,proto3" json:"upper,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*PathRange) Descriptor 

func (*PathRange) Descriptor() ([]byte, []int)

func (*PathRange) GetLower 

func (m *PathRange) GetLower() string

func (*PathRange) GetUpper 

func (m *PathRange) GetUpper() string

func (*PathRange) Marshal 

func (m *PathRange) Marshal() (dAtA []byte, err error)

func (*PathRange) MarshalLogObject 

func (x *PathRange) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PathRange) MarshalTo 

func (m *PathRange) MarshalTo(dAtA []byte) (int, error)

func (*PathRange) MarshalToSizedBuffer 

func (m *PathRange) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*PathRange) ProtoMessage 

func (*PathRange) ProtoMessage()

func (*PathRange) Reset 

func (m *PathRange) Reset()

func (*PathRange) Size 

func (m *PathRange) Size() (n int)

func (*PathRange) String 

func (m *PathRange) String() string

func (*PathRange) Unmarshal 

func (m *PathRange) Unmarshal(dAtA []byte) error

func (*PathRange) XXX_DiscardUnknown 

func (m *PathRange) XXX_DiscardUnknown()

func (*PathRange) XXX_Marshal 

func (m *PathRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*PathRange) XXX_Merge 

func (m *PathRange) XXX_Merge(src proto.Message)

func (*PathRange) XXX_Size 

func (m *PathRange) XXX_Size() int

func (*PathRange) XXX_Unmarshal 

func (m *PathRange) XXX_Unmarshal(b []byte) error

type Propagater 

type Propagater struct {
	// contains filtered or unexported fields
}

Propagater is an object that is used to propagate PFS branches at the end of a transaction. The transactionenv package provides the interface for this and will call the Run function at the end of a transaction.

func (*Propagater) DeleteBranch 

func (t *Propagater) DeleteBranch(branch *pfs.Branch)

DeleteBranch removes a branch from the list of those needing propagation if present.

func (*Propagater) PropagateBranch 

func (t *Propagater) PropagateBranch(branch *pfs.Branch) error

PropagateBranch marks a branch as needing propagation once the transaction successfully ends. This will be performed by the Run function.

func (*Propagater) Run 

func (t *Propagater) Run() error

Run performs any final tasks and cleanup tasks in the transaction, such as propagating branches

type PutFileURLTask 

type PutFileURLTask struct {
	Dst                  string   `protobuf:"bytes,1,opt,name=dst,proto3" json:"dst,omitempty"`
	Datum                string   `protobuf:"bytes,2,opt,name=datum,proto3" json:"datum,omitempty"`
	URL                  string   `protobuf:"bytes,3,opt,name=URL,proto3" json:"URL,omitempty"`
	Paths                []string `protobuf:"bytes,4,rep,name=paths,proto3" json:"paths,omitempty"`
	StartOffset          int64    `protobuf:"varint,5,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"`
	EndOffset            int64    `protobuf:"varint,7,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*PutFileURLTask) Descriptor 

func (*PutFileURLTask) Descriptor() ([]byte, []int)

func (*PutFileURLTask) GetDatum 

func (m *PutFileURLTask) GetDatum() string

func (*PutFileURLTask) GetDst 

func (m *PutFileURLTask) GetDst() string

func (*PutFileURLTask) GetEndOffset 

func (m *PutFileURLTask) GetEndOffset() int64

func (*PutFileURLTask) GetPaths 

func (m *PutFileURLTask) GetPaths() []string

func (*PutFileURLTask) GetStartOffset 

func (m *PutFileURLTask) GetStartOffset() int64

func (*PutFileURLTask) GetURL 

func (m *PutFileURLTask) GetURL() string

func (*PutFileURLTask) Marshal 

func (m *PutFileURLTask) Marshal() (dAtA []byte, err error)

func (*PutFileURLTask) MarshalLogObject 

func (x *PutFileURLTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PutFileURLTask) MarshalTo 

func (m *PutFileURLTask) MarshalTo(dAtA []byte) (int, error)

func (*PutFileURLTask) MarshalToSizedBuffer 

func (m *PutFileURLTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*PutFileURLTask) ProtoMessage 

func (*PutFileURLTask) ProtoMessage()

func (*PutFileURLTask) Reset 

func (m *PutFileURLTask) Reset()

func (*PutFileURLTask) Size 

func (m *PutFileURLTask) Size() (n int)

func (*PutFileURLTask) String 

func (m *PutFileURLTask) String() string

func (*PutFileURLTask) Unmarshal 

func (m *PutFileURLTask) Unmarshal(dAtA []byte) error

func (*PutFileURLTask) XXX_DiscardUnknown 

func (m *PutFileURLTask) XXX_DiscardUnknown()

func (*PutFileURLTask) XXX_Marshal 

func (m *PutFileURLTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*PutFileURLTask) XXX_Merge 

func (m *PutFileURLTask) XXX_Merge(src proto.Message)

func (*PutFileURLTask) XXX_Size 

func (m *PutFileURLTask) XXX_Size() int

func (*PutFileURLTask) XXX_Unmarshal 

func (m *PutFileURLTask) XXX_Unmarshal(b []byte) error

type PutFileURLTaskResult 

type PutFileURLTaskResult struct {
	Id                   string   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*PutFileURLTaskResult) Descriptor 

func (*PutFileURLTaskResult) Descriptor() ([]byte, []int)

func (*PutFileURLTaskResult) GetId 

func (m *PutFileURLTaskResult) GetId() string

func (*PutFileURLTaskResult) Marshal 

func (m *PutFileURLTaskResult) Marshal() (dAtA []byte, err error)

func (*PutFileURLTaskResult) MarshalLogObject 

func (x *PutFileURLTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PutFileURLTaskResult) MarshalTo 

func (m *PutFileURLTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*PutFileURLTaskResult) MarshalToSizedBuffer 

func (m *PutFileURLTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*PutFileURLTaskResult) ProtoMessage 

func (*PutFileURLTaskResult) ProtoMessage()

func (*PutFileURLTaskResult) Reset 

func (m *PutFileURLTaskResult) Reset()

func (*PutFileURLTaskResult) Size 

func (m *PutFileURLTaskResult) Size() (n int)

func (*PutFileURLTaskResult) String 

func (m *PutFileURLTaskResult) String() string

func (*PutFileURLTaskResult) Unmarshal 

func (m *PutFileURLTaskResult) Unmarshal(dAtA []byte) error

func (*PutFileURLTaskResult) XXX_DiscardUnknown 

func (m *PutFileURLTaskResult) XXX_DiscardUnknown()

func (*PutFileURLTaskResult) XXX_Marshal 

func (m *PutFileURLTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*PutFileURLTaskResult) XXX_Merge 

func (m *PutFileURLTaskResult) XXX_Merge(src proto.Message)

func (*PutFileURLTaskResult) XXX_Size 

func (m *PutFileURLTaskResult) XXX_Size() int

func (*PutFileURLTaskResult) XXX_Unmarshal 

func (m *PutFileURLTaskResult) XXX_Unmarshal(b []byte) error

type ShardTask 

type ShardTask struct {
	Inputs               []string   `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
	PathRange            *PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
	XXX_unrecognized     []byte     `json:"-"`
	XXX_sizecache        int32      `json:"-"`
}

func (*ShardTask) Descriptor 

func (*ShardTask) Descriptor() ([]byte, []int)

func (*ShardTask) GetInputs 

func (m *ShardTask) GetInputs() []string

func (*ShardTask) GetPathRange 

func (m *ShardTask) GetPathRange() *PathRange

func (*ShardTask) Marshal 

func (m *ShardTask) Marshal() (dAtA []byte, err error)

func (*ShardTask) MarshalLogObject 

func (x *ShardTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ShardTask) MarshalTo 

func (m *ShardTask) MarshalTo(dAtA []byte) (int, error)

func (*ShardTask) MarshalToSizedBuffer 

func (m *ShardTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ShardTask) ProtoMessage 

func (*ShardTask) ProtoMessage()

func (*ShardTask) Reset 

func (m *ShardTask) Reset()

func (*ShardTask) Size 

func (m *ShardTask) Size() (n int)

func (*ShardTask) String 

func (m *ShardTask) String() string

func (*ShardTask) Unmarshal 

func (m *ShardTask) Unmarshal(dAtA []byte) error

func (*ShardTask) XXX_DiscardUnknown 

func (m *ShardTask) XXX_DiscardUnknown()

func (*ShardTask) XXX_Marshal 

func (m *ShardTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ShardTask) XXX_Merge 

func (m *ShardTask) XXX_Merge(src proto.Message)

func (*ShardTask) XXX_Size 

func (m *ShardTask) XXX_Size() int

func (*ShardTask) XXX_Unmarshal 

func (m *ShardTask) XXX_Unmarshal(b []byte) error

type ShardTaskResult 

type ShardTaskResult struct {
	CompactTasks         []*CompactTask `protobuf:"bytes,1,rep,name=compact_tasks,json=compactTasks,proto3" json:"compact_tasks,omitempty"`
	XXX_NoUnkeyedLiteral struct{}       `json:"-"`
	XXX_unrecognized     []byte         `json:"-"`
	XXX_sizecache        int32          `json:"-"`
}

func (*ShardTaskResult) Descriptor 

func (*ShardTaskResult) Descriptor() ([]byte, []int)

func (*ShardTaskResult) GetCompactTasks 

func (m *ShardTaskResult) GetCompactTasks() []*CompactTask

func (*ShardTaskResult) Marshal 

func (m *ShardTaskResult) Marshal() (dAtA []byte, err error)

func (*ShardTaskResult) MarshalLogObject 

func (x *ShardTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ShardTaskResult) MarshalTo 

func (m *ShardTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*ShardTaskResult) MarshalToSizedBuffer 

func (m *ShardTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ShardTaskResult) ProtoMessage 

func (*ShardTaskResult) ProtoMessage()

func (*ShardTaskResult) Reset 

func (m *ShardTaskResult) Reset()

func (*ShardTaskResult) Size 

func (m *ShardTaskResult) Size() (n int)

func (*ShardTaskResult) String 

func (m *ShardTaskResult) String() string

func (*ShardTaskResult) Unmarshal 

func (m *ShardTaskResult) Unmarshal(dAtA []byte) error

func (*ShardTaskResult) XXX_DiscardUnknown 

func (m *ShardTaskResult) XXX_DiscardUnknown()

func (*ShardTaskResult) XXX_Marshal 

func (m *ShardTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ShardTaskResult) XXX_Merge 

func (m *ShardTaskResult) XXX_Merge(src proto.Message)

func (*ShardTaskResult) XXX_Size 

func (m *ShardTaskResult) XXX_Size() int

func (*ShardTaskResult) XXX_Unmarshal 

func (m *ShardTaskResult) XXX_Unmarshal(b []byte) error

type Source 

type Source interface {
	// Iterate calls cb for each File in the underlying fileset.FileSet, with a FileInfo computed
	// during iteration, and the File.
	Iterate(ctx context.Context, cb func(*pfs.FileInfo, fileset.File) error) error
}

Source iterates over FileInfos generated from a fileset.FileSet

func NewErrOnEmpty 

func NewErrOnEmpty(s Source, err error) Source

NewErrOnEmpty causes iterate to return a not found error if there are no items to iterate over

func NewSource 

func NewSource(commitInfo *pfs.CommitInfo, fs fileset.FileSet, opts ...SourceOption) Source

NewSource creates a Source which emits FileInfos with the information from commit, and the entries return from fileSet.

type SourceOption 

type SourceOption func(*sourceConfig)

SourceOption configures a source.

func WithDatum 

func WithDatum(datum string) SourceOption

func WithFilter 

func WithFilter(filter func(fileset.FileSet) fileset.FileSet) SourceOption

WithFilter applies a filter to the file set after it has been set up by the source.

func WithPathRange 

func WithPathRange(pathRange *pfs.PathRange) SourceOption

func WithPrefix 

func WithPrefix(prefix string) SourceOption

type ValidateTask 

type ValidateTask struct {
	Id                   string     `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	PathRange            *PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
	XXX_unrecognized     []byte     `json:"-"`
	XXX_sizecache        int32      `json:"-"`
}

func (*ValidateTask) Descriptor 

func (*ValidateTask) Descriptor() ([]byte, []int)

func (*ValidateTask) GetId 

func (m *ValidateTask) GetId() string

func (*ValidateTask) GetPathRange 

func (m *ValidateTask) GetPathRange() *PathRange

func (*ValidateTask) Marshal 

func (m *ValidateTask) Marshal() (dAtA []byte, err error)

func (*ValidateTask) MarshalLogObject 

func (x *ValidateTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ValidateTask) MarshalTo 

func (m *ValidateTask) MarshalTo(dAtA []byte) (int, error)

func (*ValidateTask) MarshalToSizedBuffer 

func (m *ValidateTask) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ValidateTask) ProtoMessage 

func (*ValidateTask) ProtoMessage()

func (*ValidateTask) Reset 

func (m *ValidateTask) Reset()

func (*ValidateTask) Size 

func (m *ValidateTask) Size() (n int)

func (*ValidateTask) String 

func (m *ValidateTask) String() string

func (*ValidateTask) Unmarshal 

func (m *ValidateTask) Unmarshal(dAtA []byte) error

func (*ValidateTask) XXX_DiscardUnknown 

func (m *ValidateTask) XXX_DiscardUnknown()

func (*ValidateTask) XXX_Marshal 

func (m *ValidateTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ValidateTask) XXX_Merge 

func (m *ValidateTask) XXX_Merge(src proto.Message)

func (*ValidateTask) XXX_Size 

func (m *ValidateTask) XXX_Size() int

func (*ValidateTask) XXX_Unmarshal 

func (m *ValidateTask) XXX_Unmarshal(b []byte) error

type ValidateTaskResult 

type ValidateTaskResult struct {
	First                *index.Index `protobuf:"bytes,1,opt,name=first,proto3" json:"first,omitempty"`
	Last                 *index.Index `protobuf:"bytes,2,opt,name=last,proto3" json:"last,omitempty"`
	Error                string       `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"`
	SizeBytes            int64        `protobuf:"varint,4,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
	XXX_NoUnkeyedLiteral struct{}     `json:"-"`
	XXX_unrecognized     []byte       `json:"-"`
	XXX_sizecache        int32        `json:"-"`
}

func (*ValidateTaskResult) Descriptor 

func (*ValidateTaskResult) Descriptor() ([]byte, []int)

func (*ValidateTaskResult) GetError 

func (m *ValidateTaskResult) GetError() string

func (*ValidateTaskResult) GetFirst 

func (m *ValidateTaskResult) GetFirst() *index.Index

func (*ValidateTaskResult) GetLast 

func (m *ValidateTaskResult) GetLast() *index.Index

func (*ValidateTaskResult) GetSizeBytes 

func (m *ValidateTaskResult) GetSizeBytes() int64

func (*ValidateTaskResult) Marshal 

func (m *ValidateTaskResult) Marshal() (dAtA []byte, err error)

func (*ValidateTaskResult) MarshalLogObject 

func (x *ValidateTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ValidateTaskResult) MarshalTo 

func (m *ValidateTaskResult) MarshalTo(dAtA []byte) (int, error)

func (*ValidateTaskResult) MarshalToSizedBuffer 

func (m *ValidateTaskResult) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ValidateTaskResult) ProtoMessage 

func (*ValidateTaskResult) ProtoMessage()

func (*ValidateTaskResult) Reset 

func (m *ValidateTaskResult) Reset()

func (*ValidateTaskResult) Size 

func (m *ValidateTaskResult) Size() (n int)

func (*ValidateTaskResult) String 

func (m *ValidateTaskResult) String() string

func (*ValidateTaskResult) Unmarshal 

func (m *ValidateTaskResult) Unmarshal(dAtA []byte) error

func (*ValidateTaskResult) XXX_DiscardUnknown 

func (m *ValidateTaskResult) XXX_DiscardUnknown()

func (*ValidateTaskResult) XXX_Marshal 

func (m *ValidateTaskResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ValidateTaskResult) XXX_Merge 

func (m *ValidateTaskResult) XXX_Merge(src proto.Message)

func (*ValidateTaskResult) XXX_Size 

func (m *ValidateTaskResult) XXX_Size() int

func (*ValidateTaskResult) XXX_Unmarshal 

func (m *ValidateTaskResult) XXX_Unmarshal(b []byte) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.