server

package
v2.7.0-nightly.20230726 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 25, 2023 License: Apache-2.0 Imports: 95 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	StorageTaskNamespace = "storage"
)
View Source 
const (
	URLTaskNamespace = "url"
)

Variables

View Source 
var File_server_pfs_server_pfsserver_proto protoreflect.FileDescriptor

Functions

func IsPermissionError 

func IsPermissionError(err error) bool

IsPermissionError returns true if a given error is a permission error.

func NewAPIServer 

func NewAPIServer(env Env) (pfsserver.APIServer, error)

NewAPIServer creates an APIServer.

func NewPachwAPIServer 

func NewPachwAPIServer(env Env) (pfsserver.APIServer, error)

NewPachwAPIServer is used when running pachd in Pachw Mode. In Pachw Mode, a pachd instance processes storage and URl related tasks via the task service.

func NewSidecarAPIServer 

func NewSidecarAPIServer(env Env) (pfsserver.APIServer, error)

func TopologicalSort 

func TopologicalSort(cis []*pfs.CommitInfo) []*pfs.CommitInfo

TopologicalSort sorts a slice of commit infos topologically based on their provenance

func ValidateFilename 

func ValidateFilename(p string) error

Types

type CommitEvent 

type CommitEvent struct {
	Err   error
	Value *pfs.CommitInfo
}

CommitEvent is an event that contains a CommitInfo or an error

type CommitStream 

type CommitStream interface {
	Stream() <-chan CommitEvent
	Close()
}

CommitStream is a stream of CommitInfos

type CompactTask 

type CompactTask struct {
	Inputs    []string   `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
	PathRange *PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	// contains filtered or unexported fields
}

func (*CompactTask) Descriptor deprecated 

func (*CompactTask) Descriptor() ([]byte, []int)

Deprecated: Use CompactTask.ProtoReflect.Descriptor instead.

func (*CompactTask) GetInputs 

func (x *CompactTask) GetInputs() []string

func (*CompactTask) GetPathRange 

func (x *CompactTask) GetPathRange() *PathRange

func (*CompactTask) MarshalLogObject 

func (x *CompactTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*CompactTask) ProtoMessage 

func (*CompactTask) ProtoMessage()

func (*CompactTask) ProtoReflect added in v2.7.0 

func (x *CompactTask) ProtoReflect() protoreflect.Message

func (*CompactTask) Reset 

func (x *CompactTask) Reset()

func (*CompactTask) String 

func (x *CompactTask) String() string

type CompactTaskResult 

type CompactTaskResult struct {
	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	// contains filtered or unexported fields
}

func (*CompactTaskResult) Descriptor deprecated 

func (*CompactTaskResult) Descriptor() ([]byte, []int)

Deprecated: Use CompactTaskResult.ProtoReflect.Descriptor instead.

func (*CompactTaskResult) GetId 

func (x *CompactTaskResult) GetId() string

func (*CompactTaskResult) MarshalLogObject 

func (x *CompactTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*CompactTaskResult) ProtoMessage 

func (*CompactTaskResult) ProtoMessage()

func (*CompactTaskResult) ProtoReflect added in v2.7.0 

func (x *CompactTaskResult) ProtoReflect() protoreflect.Message

func (*CompactTaskResult) Reset 

func (x *CompactTaskResult) Reset()

func (*CompactTaskResult) String 

func (x *CompactTaskResult) String() string

type ConcatTask 

type ConcatTask struct {
	Inputs []string `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
	// contains filtered or unexported fields
}

func (*ConcatTask) Descriptor deprecated 

func (*ConcatTask) Descriptor() ([]byte, []int)

Deprecated: Use ConcatTask.ProtoReflect.Descriptor instead.

func (*ConcatTask) GetInputs 

func (x *ConcatTask) GetInputs() []string

func (*ConcatTask) MarshalLogObject 

func (x *ConcatTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ConcatTask) ProtoMessage 

func (*ConcatTask) ProtoMessage()

func (*ConcatTask) ProtoReflect added in v2.7.0 

func (x *ConcatTask) ProtoReflect() protoreflect.Message

func (*ConcatTask) Reset 

func (x *ConcatTask) Reset()

func (*ConcatTask) String 

func (x *ConcatTask) String() string

type ConcatTaskResult 

type ConcatTaskResult struct {
	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	// contains filtered or unexported fields
}

func (*ConcatTaskResult) Descriptor deprecated 

func (*ConcatTaskResult) Descriptor() ([]byte, []int)

Deprecated: Use ConcatTaskResult.ProtoReflect.Descriptor instead.

func (*ConcatTaskResult) GetId 

func (x *ConcatTaskResult) GetId() string

func (*ConcatTaskResult) MarshalLogObject 

func (x *ConcatTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ConcatTaskResult) ProtoMessage 

func (*ConcatTaskResult) ProtoMessage()

func (*ConcatTaskResult) ProtoReflect added in v2.7.0 

func (x *ConcatTaskResult) ProtoReflect() protoreflect.Message

func (*ConcatTaskResult) Reset 

func (x *ConcatTaskResult) Reset()

func (*ConcatTaskResult) String 

func (x *ConcatTaskResult) String() string

type Differ 

type Differ struct {
	// contains filtered or unexported fields
}

Differ compares two sources and iterates over the items that are not equal.

func NewDiffer 

func NewDiffer(a, b Source) *Differ

NewDiffer creates an iterator over the differences between Sources a and b

func (*Differ) Iterate 

func (d *Differ) Iterate(ctx context.Context, cb func(aFi, bFi *pfs.FileInfo) error) error

Iterate compares the entries from `a` and `b` path wise. If one side is missing a path, cb is called with the info for the side that has the path If both sides have a path, but the content is different, cb is called with the info for both sides at once. If both sides have a path, and the content is the same, cb is not called. The info is not part of the diff.

type Env 

type Env struct {
	ObjectClient obj.Client
	DB           *pachsql.DB
	EtcdPrefix   string
	EtcdClient   *etcd.Client
	TaskService  task.Service
	TxnEnv       *txnenv.TransactionEnv
	Listener     col.PostgresListener

	AuthServer authserver.APIServer
	// TODO: a reasonable repo metadata solution would let us get rid of this circular dependency
	// permissions might also work.
	GetPPSServer func() ppsserver.APIServer
	// TODO: remove this, the load tests need a pachClient
	GetPachClient func(ctx context.Context) *client.APIClient

	BackgroundContext context.Context
	StorageConfig     pachconfig.StorageConfiguration
	PachwInSidecar    bool
}

Env is the dependencies needed to run the PFS API server

func EnvFromServiceEnv 

func EnvFromServiceEnv(env serviceenv.ServiceEnv, txnEnv *txnenv.TransactionEnv) (*Env, error)

type ErrBranchCommitProvenanceMismatch 

type ErrBranchCommitProvenanceMismatch struct {
	Branch       *pfs.Branch
	ParentBranch *pfs.Branch
}

ErrBranchCommitProvenanceMismatch occurs when the head commit of one of the parents of the branch is not found in the direct provenance commits of the head of the branch

func (ErrBranchCommitProvenanceMismatch) Error 

func (e ErrBranchCommitProvenanceMismatch) Error() string

type ErrBranchInfoNotFound 

type ErrBranchInfoNotFound struct {
	Branch *pfs.Branch
}

ErrBranchInfoNotFound Branch info could not be found. Typically because of an incomplete deletion of a branch. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrBranchInfoNotFound) Error 

func (e ErrBranchInfoNotFound) Error() string

type ErrBranchProvenanceTransitivity 

type ErrBranchProvenanceTransitivity struct {
	BranchInfo     *pfs.BranchInfo
	FullProvenance []*pfs.Branch
}

ErrBranchProvenanceTransitivity Branch provenance is not transitively closed. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrBranchProvenanceTransitivity) Error 

func (e ErrBranchProvenanceTransitivity) Error() string

type ErrBranchSubvenanceTransitivity 

type ErrBranchSubvenanceTransitivity struct {
	BranchInfo        *pfs.BranchInfo
	MissingSubvenance *pfs.Branch
}

func (ErrBranchSubvenanceTransitivity) Error 

func (e ErrBranchSubvenanceTransitivity) Error() string

type ErrCommitAncestryBroken 

type ErrCommitAncestryBroken struct {
	Parent *pfs.Commit
	Child  *pfs.Commit
}

ErrCommitAncestryBroken indicates that a parent and child commit disagree on their relationship. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrCommitAncestryBroken) Error 

func (e ErrCommitAncestryBroken) Error() string

type ErrCommitInfoNotFound 

type ErrCommitInfoNotFound struct {
	Location string
	Commit   *pfs.Commit
}

ErrCommitInfoNotFound Commit info could not be found. Typically because of an incomplete deletion of a commit. This struct contains all the information that was used to demonstrate that this invariant is not being satisfied.

func (ErrCommitInfoNotFound) Error 

func (e ErrCommitInfoNotFound) Error() string

type ErrZombieData 

type ErrZombieData struct {
	Commit *pfs.Commit
	ID     string
}

func (ErrZombieData) Error 

func (e ErrZombieData) Error() string

type GetFileURLTask 

type GetFileURLTask struct {
	URL       string         `protobuf:"bytes,1,opt,name=URL,proto3" json:"URL,omitempty"`
	File      *pfs.File      `protobuf:"bytes,2,opt,name=file,proto3" json:"file,omitempty"`
	PathRange *pfs.PathRange `protobuf:"bytes,3,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	// contains filtered or unexported fields
}

func (*GetFileURLTask) Descriptor deprecated 

func (*GetFileURLTask) Descriptor() ([]byte, []int)

Deprecated: Use GetFileURLTask.ProtoReflect.Descriptor instead.

func (*GetFileURLTask) GetFile 

func (x *GetFileURLTask) GetFile() *pfs.File

func (*GetFileURLTask) GetPathRange 

func (x *GetFileURLTask) GetPathRange() *pfs.PathRange

func (*GetFileURLTask) GetURL 

func (x *GetFileURLTask) GetURL() string

func (*GetFileURLTask) MarshalLogObject 

func (x *GetFileURLTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*GetFileURLTask) ProtoMessage 

func (*GetFileURLTask) ProtoMessage()

func (*GetFileURLTask) ProtoReflect added in v2.7.0 

func (x *GetFileURLTask) ProtoReflect() protoreflect.Message

func (*GetFileURLTask) Reset 

func (x *GetFileURLTask) Reset()

func (*GetFileURLTask) String 

func (x *GetFileURLTask) String() string

type GetFileURLTaskResult 

type GetFileURLTaskResult struct {
	// contains filtered or unexported fields
}

func (*GetFileURLTaskResult) Descriptor deprecated 

func (*GetFileURLTaskResult) Descriptor() ([]byte, []int)

Deprecated: Use GetFileURLTaskResult.ProtoReflect.Descriptor instead.

func (*GetFileURLTaskResult) MarshalLogObject 

func (x *GetFileURLTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*GetFileURLTaskResult) ProtoMessage 

func (*GetFileURLTaskResult) ProtoMessage()

func (*GetFileURLTaskResult) ProtoReflect added in v2.7.0 

func (x *GetFileURLTaskResult) ProtoReflect() protoreflect.Message

func (*GetFileURLTaskResult) Reset 

func (x *GetFileURLTaskResult) Reset()

func (*GetFileURLTaskResult) String 

func (x *GetFileURLTaskResult) String() string

type PathRange 

type PathRange struct {
	Lower string `protobuf:"bytes,1,opt,name=lower,proto3" json:"lower,omitempty"`
	Upper string `protobuf:"bytes,2,opt,name=upper,proto3" json:"upper,omitempty"`
	// contains filtered or unexported fields
}

func (*PathRange) Descriptor deprecated 

func (*PathRange) Descriptor() ([]byte, []int)

Deprecated: Use PathRange.ProtoReflect.Descriptor instead.

func (*PathRange) GetLower 

func (x *PathRange) GetLower() string

func (*PathRange) GetUpper 

func (x *PathRange) GetUpper() string

func (*PathRange) MarshalLogObject 

func (x *PathRange) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PathRange) ProtoMessage 

func (*PathRange) ProtoMessage()

func (*PathRange) ProtoReflect added in v2.7.0 

func (x *PathRange) ProtoReflect() protoreflect.Message

func (*PathRange) Reset 

func (x *PathRange) Reset()

func (*PathRange) String 

func (x *PathRange) String() string

type Propagater 

type Propagater struct {
	// contains filtered or unexported fields
}

Propagater is an object that is used to propagate PFS branches at the end of a transaction. The transactionenv package provides the interface for this and will call the Run function at the end of a transaction.

func (*Propagater) DeleteBranch 

func (t *Propagater) DeleteBranch(branch *pfs.Branch)

DeleteBranch removes a branch from the list of those needing propagation if present.

func (*Propagater) PropagateBranch 

func (t *Propagater) PropagateBranch(branch *pfs.Branch) error

PropagateBranch marks a branch as needing propagation once the transaction successfully ends. This will be performed by the Run function.

func (*Propagater) Run 

func (t *Propagater) Run() error

Run performs any final tasks and cleanup tasks in the transaction, such as propagating branches

type PutFileURLTask 

type PutFileURLTask struct {
	Dst         string   `protobuf:"bytes,1,opt,name=dst,proto3" json:"dst,omitempty"`
	Datum       string   `protobuf:"bytes,2,opt,name=datum,proto3" json:"datum,omitempty"`
	URL         string   `protobuf:"bytes,3,opt,name=URL,proto3" json:"URL,omitempty"`
	Paths       []string `protobuf:"bytes,4,rep,name=paths,proto3" json:"paths,omitempty"`
	StartOffset int64    `protobuf:"varint,5,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"`
	EndOffset   int64    `protobuf:"varint,7,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"`
	// contains filtered or unexported fields
}

func (*PutFileURLTask) Descriptor deprecated 

func (*PutFileURLTask) Descriptor() ([]byte, []int)

Deprecated: Use PutFileURLTask.ProtoReflect.Descriptor instead.

func (*PutFileURLTask) GetDatum 

func (x *PutFileURLTask) GetDatum() string

func (*PutFileURLTask) GetDst 

func (x *PutFileURLTask) GetDst() string

func (*PutFileURLTask) GetEndOffset 

func (x *PutFileURLTask) GetEndOffset() int64

func (*PutFileURLTask) GetPaths 

func (x *PutFileURLTask) GetPaths() []string

func (*PutFileURLTask) GetStartOffset 

func (x *PutFileURLTask) GetStartOffset() int64

func (*PutFileURLTask) GetURL 

func (x *PutFileURLTask) GetURL() string

func (*PutFileURLTask) MarshalLogObject 

func (x *PutFileURLTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PutFileURLTask) ProtoMessage 

func (*PutFileURLTask) ProtoMessage()

func (*PutFileURLTask) ProtoReflect added in v2.7.0 

func (x *PutFileURLTask) ProtoReflect() protoreflect.Message

func (*PutFileURLTask) Reset 

func (x *PutFileURLTask) Reset()

func (*PutFileURLTask) String 

func (x *PutFileURLTask) String() string

type PutFileURLTaskResult 

type PutFileURLTaskResult struct {
	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	// contains filtered or unexported fields
}

func (*PutFileURLTaskResult) Descriptor deprecated 

func (*PutFileURLTaskResult) Descriptor() ([]byte, []int)

Deprecated: Use PutFileURLTaskResult.ProtoReflect.Descriptor instead.

func (*PutFileURLTaskResult) GetId 

func (x *PutFileURLTaskResult) GetId() string

func (*PutFileURLTaskResult) MarshalLogObject 

func (x *PutFileURLTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*PutFileURLTaskResult) ProtoMessage 

func (*PutFileURLTaskResult) ProtoMessage()

func (*PutFileURLTaskResult) ProtoReflect added in v2.7.0 

func (x *PutFileURLTaskResult) ProtoReflect() protoreflect.Message

func (*PutFileURLTaskResult) Reset 

func (x *PutFileURLTaskResult) Reset()

func (*PutFileURLTaskResult) String 

func (x *PutFileURLTaskResult) String() string

type ShardTask 

type ShardTask struct {
	Inputs    []string   `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"`
	PathRange *PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	// contains filtered or unexported fields
}

func (*ShardTask) Descriptor deprecated 

func (*ShardTask) Descriptor() ([]byte, []int)

Deprecated: Use ShardTask.ProtoReflect.Descriptor instead.

func (*ShardTask) GetInputs 

func (x *ShardTask) GetInputs() []string

func (*ShardTask) GetPathRange 

func (x *ShardTask) GetPathRange() *PathRange

func (*ShardTask) MarshalLogObject 

func (x *ShardTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ShardTask) ProtoMessage 

func (*ShardTask) ProtoMessage()

func (*ShardTask) ProtoReflect added in v2.7.0 

func (x *ShardTask) ProtoReflect() protoreflect.Message

func (*ShardTask) Reset 

func (x *ShardTask) Reset()

func (*ShardTask) String 

func (x *ShardTask) String() string

type ShardTaskResult 

type ShardTaskResult struct {
	CompactTasks []*CompactTask `protobuf:"bytes,1,rep,name=compact_tasks,json=compactTasks,proto3" json:"compact_tasks,omitempty"`
	// contains filtered or unexported fields
}

func (*ShardTaskResult) Descriptor deprecated 

func (*ShardTaskResult) Descriptor() ([]byte, []int)

Deprecated: Use ShardTaskResult.ProtoReflect.Descriptor instead.

func (*ShardTaskResult) GetCompactTasks 

func (x *ShardTaskResult) GetCompactTasks() []*CompactTask

func (*ShardTaskResult) MarshalLogObject 

func (x *ShardTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ShardTaskResult) ProtoMessage 

func (*ShardTaskResult) ProtoMessage()

func (*ShardTaskResult) ProtoReflect added in v2.7.0 

func (x *ShardTaskResult) ProtoReflect() protoreflect.Message

func (*ShardTaskResult) Reset 

func (x *ShardTaskResult) Reset()

func (*ShardTaskResult) String 

func (x *ShardTaskResult) String() string

type Source 

type Source interface {
	// Iterate calls cb for each File in the underlying fileset.FileSet, with a FileInfo computed
	// during iteration, and the File.
	Iterate(ctx context.Context, cb func(*pfs.FileInfo, fileset.File) error) error
}

Source iterates over FileInfos generated from a fileset.FileSet

func NewErrOnEmpty 

func NewErrOnEmpty(s Source, err error) Source

NewErrOnEmpty causes iterate to return a not found error if there are no items to iterate over

func NewSource 

func NewSource(commitInfo *pfs.CommitInfo, fs fileset.FileSet, opts ...SourceOption) Source

NewSource creates a Source which emits FileInfos with the information from commit, and the entries return from fileSet.

type SourceOption 

type SourceOption func(*sourceConfig)

SourceOption configures a source.

func WithDatum 

func WithDatum(datum string) SourceOption

func WithFilter 

func WithFilter(filter func(fileset.FileSet) fileset.FileSet) SourceOption

WithFilter applies a filter to the file set after it has been set up by the source.

func WithPathRange 

func WithPathRange(pathRange *pfs.PathRange) SourceOption

func WithPrefix 

func WithPrefix(prefix string) SourceOption

type ValidateTask 

type ValidateTask struct {
	Id        string     `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	PathRange *PathRange `protobuf:"bytes,2,opt,name=path_range,json=pathRange,proto3" json:"path_range,omitempty"`
	// contains filtered or unexported fields
}

func (*ValidateTask) Descriptor deprecated 

func (*ValidateTask) Descriptor() ([]byte, []int)

Deprecated: Use ValidateTask.ProtoReflect.Descriptor instead.

func (*ValidateTask) GetId 

func (x *ValidateTask) GetId() string

func (*ValidateTask) GetPathRange 

func (x *ValidateTask) GetPathRange() *PathRange

func (*ValidateTask) MarshalLogObject 

func (x *ValidateTask) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ValidateTask) ProtoMessage 

func (*ValidateTask) ProtoMessage()

func (*ValidateTask) ProtoReflect added in v2.7.0 

func (x *ValidateTask) ProtoReflect() protoreflect.Message

func (*ValidateTask) Reset 

func (x *ValidateTask) Reset()

func (*ValidateTask) String 

func (x *ValidateTask) String() string

type ValidateTaskResult 

type ValidateTaskResult struct {
	First     *index.Index `protobuf:"bytes,1,opt,name=first,proto3" json:"first,omitempty"`
	Last      *index.Index `protobuf:"bytes,2,opt,name=last,proto3" json:"last,omitempty"`
	Error     string       `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"`
	SizeBytes int64        `protobuf:"varint,4,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
	// contains filtered or unexported fields
}

func (*ValidateTaskResult) Descriptor deprecated 

func (*ValidateTaskResult) Descriptor() ([]byte, []int)

Deprecated: Use ValidateTaskResult.ProtoReflect.Descriptor instead.

func (*ValidateTaskResult) GetError 

func (x *ValidateTaskResult) GetError() string

func (*ValidateTaskResult) GetFirst 

func (x *ValidateTaskResult) GetFirst() *index.Index

func (*ValidateTaskResult) GetLast 

func (x *ValidateTaskResult) GetLast() *index.Index

func (*ValidateTaskResult) GetSizeBytes 

func (x *ValidateTaskResult) GetSizeBytes() int64

func (*ValidateTaskResult) MarshalLogObject 

func (x *ValidateTaskResult) MarshalLogObject(enc zapcore.ObjectEncoder) error

func (*ValidateTaskResult) ProtoMessage 

func (*ValidateTaskResult) ProtoMessage()

func (*ValidateTaskResult) ProtoReflect added in v2.7.0 

func (x *ValidateTaskResult) ProtoReflect() protoreflect.Message

func (*ValidateTaskResult) Reset 

func (x *ValidateTaskResult) Reset()

func (*ValidateTaskResult) String 

func (x *ValidateTaskResult) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.