catalog

package
v1.20.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 16, 2024 License: Apache-2.0 Imports: 51 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultActionsCacheSize   = 100
	DefaultActionsCacheExpiry = 5 * time.Second
	DefaultActionsCacheJitter = DefaultActionsCacheExpiry / 2
)
View Source 
const (
	NumberOfParentsOfNonMergeCommit = 1

	DumpRefsTaskIDPrefix    = "DR"
	RestoreRefsTaskIDPrefix = "RR"

	TaskExpiryTime = 24 * time.Hour
)
View Source 
const (
	RangeFSName     = "range"
	MetaRangeFSName = "meta-range"
)
View Source 
const (
	ListRepositoriesLimitMax = 1000
	ListBranchesLimitMax     = 1000
	ListTagsLimitMax         = 1000
	DiffLimitMax             = 1000
	ListEntriesLimitMax      = 10000
)
View Source 
const (
	ImportPathTypePrefix = "common_prefix"
	ImportPathTypeObject = "object"
)
View Source 
const (
	DefaultContentType = "application/octet-stream"
)
View Source 
const (
	DefaultPathDelimiter = "/"
)
View Source 
const ImportCanceled = "Canceled"
View Source 
const (
	MaxPathLength = 1024
)

Variables

View Source 
var (
	Entry_AddressType_name = map[int32]string{
		0: "BY_PREFIX_DEPRECATED",
		1: "RELATIVE",
		2: "FULL",
	}
	Entry_AddressType_value = map[string]int32{
		"BY_PREFIX_DEPRECATED": 0,
		"RELATIVE":             1,
		"FULL":                 2,
	}
)

Enum value maps for Entry_AddressType.

View Source 
var (
	ErrUnknownDiffType          = errors.New("unknown graveler difference type")
	ErrPathRequiredValue        = fmt.Errorf("missing path: %w", graveler.ErrRequiredValue)
	ErrInvalidMetadataSrcFormat = errors.New("invalid metadata src format")
	ErrExpired                  = errors.New("expired from storage")

	// ErrItClosed is used to determine the reason for the end of the walk
	ErrItClosed = errors.New("iterator closed")

	ErrFeatureNotSupported = errors.New("feature not supported")
	ErrNonEmptyRepository  = errors.New("non empty repository")
)

Define errors we raise from this package - do not convert underlying errors, optionally wrap if needed to consolidate

View Source 
var ErrImportClosed = errors.New("import closed")
View Source 
var File_catalog_catalog_proto protoreflect.FileDescriptor
View Source 
var ValidatePathOptional = validator.MakeValidateOptional(ValidatePath)

Functions

func ContentTypeOrDefault added in v0.53.0 

func ContentTypeOrDefault(ct string) string

func EntryToValue 

func EntryToValue(entry *Entry) (*graveler.Value, error)

func GetStartPos added in v0.42.0 

func GetStartPos(prefix, after, delimiter string) string

GetStartPos returns a key that SeekGE will transform to a place start iterating on all elements in 

the keys that start with 'prefix' after 'after' and taking 'delimiter' into account

func GetTaskStatus added in v1.3.0 

func GetTaskStatus(ctx context.Context, kvStore kv.Store, repository *graveler.RepositoryRecord, taskID string, statusMsg protoreflect.ProtoMessage) error

func IsTaskID added in v1.3.0 

func IsTaskID(prefix, taskID string) bool

func MustEntryToValue 

func MustEntryToValue(entry *Entry) *graveler.Value

func NewEntryToValueIterator 

func NewEntryToValueIterator(it EntryIterator) *entryValueIterator

func NewTaskID added in v1.3.0 

func NewTaskID(prefix string) string

func NewValueToEntryIterator 

func NewValueToEntryIterator(it graveler.ValueIterator) *valueEntryIterator

func NewWalkEntryIterator added in v0.65.0 

func NewWalkEntryIterator(ctx context.Context, walker *store.WalkerWrapper, sourceType ImportPathType, destination, after, continuationToken string) (*walkEntryIterator, error)

func TaskPath added in v1.3.0 

func TaskPath(key string) string

func UpdateTaskStatus added in v1.3.0 

func UpdateTaskStatus(ctx context.Context, kvStore kv.Store, repository *graveler.RepositoryRecord, taskID string, statusMsg protoreflect.ProtoMessage) error

func ValidatePath 

func ValidatePath(v interface{}) error

Types

type ActionsCacheConfig added in v0.84.0 

type ActionsCacheConfig struct {
	Size   int
	Expiry time.Duration
	Jitter time.Duration
}

type ActionsOutputWriter 

type ActionsOutputWriter struct {
	// contains filtered or unexported fields
}

func NewActionsOutputWriter 

func NewActionsOutputWriter(blockAdapter block.Adapter) *ActionsOutputWriter

func (*ActionsOutputWriter) OutputWrite 

func (o *ActionsOutputWriter) OutputWrite(ctx context.Context, storageNamespace, name string, reader io.Reader, size int64) error

type ActionsSource 

type ActionsSource struct {
	// contains filtered or unexported fields
}

func NewActionsSource 

func NewActionsSource(catalog *Catalog) *ActionsSource

func (*ActionsSource) List 

func (s *ActionsSource) List(ctx context.Context, record graveler.HookRecord) ([]string, error)

func (*ActionsSource) Load 

func (s *ActionsSource) Load(ctx context.Context, record graveler.HookRecord, name string) ([]byte, error)

type AddressType added in v0.40.0 

type AddressType int32

AddressType is the type of an entry address 

const (
	// Deprecated: indicates that the address might be relative or full.
	// Used only for backward compatibility and should not be used for creating entries.
	AddressTypeByPrefixDeprecated AddressType = 0

	// AddressTypeRelative indicates that the address is relative to the storage namespace.
	// For example: "foo/bar"
	AddressTypeRelative AddressType = 1

	// AddressTypeFull indicates that the address is the full address of the object in the object store.
	// For example: "s3://bucket/foo/bar"
	AddressTypeFull AddressType = 2
)

func (AddressType) ToIdentifierType added in v0.40.0 

func (at AddressType) ToIdentifierType() block.IdentifierType

type Branch 

type Branch struct {
	Name      string
	Reference string
}

type Catalog 

type Catalog struct {
	BlockAdapter block.Adapter
	Store        Store

	PathProvider      *upload.PathPartitionProvider
	BackgroundLimiter ratelimit.Limiter
	KVStore           kv.Store
	KVStoreLimited    kv.Store

	UGCPrepareMaxFileSize int64
	UGCPrepareInterval    time.Duration
	// contains filtered or unexported fields
}

func New 

func New(ctx context.Context, cfg Config) (*Catalog, error)

func (*Catalog) BranchExists 

func (c *Catalog) BranchExists(ctx context.Context, repositoryID string, branch string) (bool, error)

func (*Catalog) CancelImport added in v0.102.0 

func (c *Catalog) CancelImport(ctx context.Context, repositoryID, importID string) error

func (*Catalog) CherryPick added in v0.97.0 

func (c *Catalog) CherryPick(ctx context.Context, repositoryID string, branch string, params CherryPickParams, opts ...graveler.SetOptionsFunc) (*CommitLog, error)

func (*Catalog) Close 

func (c *Catalog) Close() error

func (*Catalog) Commit 

func (c *Catalog) Commit(ctx context.Context, repositoryID, branch, message, committer string, metadata Metadata, date *int64, sourceMetarange *string, allowEmpty bool, opts ...graveler.SetOptionsFunc) (*CommitLog, error)

func (*Catalog) Compare 

func (c *Catalog) Compare(ctx context.Context, repositoryID, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)

func (*Catalog) CopyEntry added in v0.90.1 

func (c *Catalog) CopyEntry(ctx context.Context, srcRepository, srcRef, srcPath, destRepository, destBranch, destPath string, opts ...graveler.SetOptionsFunc) (*DBEntry, error)

CopyEntry copy entry information by using the block adapter to make a copy of the data to a new physical address.

func (*Catalog) CreateBareRepository 

func (c *Catalog) CreateBareRepository(ctx context.Context, repository string, storageNamespace string, defaultBranchID string, readOnly bool) (*Repository, error)

CreateBareRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with no initial branch or commit defaultBranchID will point to a non-existent branch on creation, it is up to the caller to eventually create it.

func (*Catalog) CreateBranch 

func (c *Catalog) CreateBranch(ctx context.Context, repositoryID string, branch string, sourceBranch string, opts ...graveler.SetOptionsFunc) (*CommitLog, error)

func (*Catalog) CreateCommitRecord added in v1.9.0 

func (c *Catalog) CreateCommitRecord(ctx context.Context, repositoryID string, commitID string, version int, committer string, message string, metaRangeID string, creationDate int64, parents []string, metadata map[string]string, generation int, opts ...graveler.SetOptionsFunc) error

func (*Catalog) CreateEntry 

func (c *Catalog) CreateEntry(ctx context.Context, repositoryID string, branch string, entry DBEntry, opts ...graveler.SetOptionsFunc) error

func (*Catalog) CreateRepository 

func (c *Catalog) CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string, readOnly bool) (*Repository, error)

CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch'

func (*Catalog) CreateTag 

func (c *Catalog) CreateTag(ctx context.Context, repositoryID string, tagID string, ref string, opts ...graveler.SetOptionsFunc) (string, error)

func (*Catalog) DeleteBranch 

func (c *Catalog) DeleteBranch(ctx context.Context, repositoryID string, branch string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) DeleteEntries added in v0.82.0 

func (c *Catalog) DeleteEntries(ctx context.Context, repositoryID string, branch string, paths []string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) DeleteEntry 

func (c *Catalog) DeleteEntry(ctx context.Context, repositoryID string, branch string, path string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) DeleteExpiredImports added in v0.102.0 

func (c *Catalog) DeleteExpiredImports(ctx context.Context)

func (*Catalog) DeleteExpiredLinkAddresses added in v0.90.0 

func (c *Catalog) DeleteExpiredLinkAddresses(ctx context.Context)

func (*Catalog) DeleteExpiredTasks added in v1.3.0 

func (c *Catalog) DeleteExpiredTasks(ctx context.Context)

func (*Catalog) DeleteRepository 

func (c *Catalog) DeleteRepository(ctx context.Context, repository string, opts ...graveler.SetOptionsFunc) error

DeleteRepository delete a repository

func (*Catalog) DeleteRepositoryMetadata added in v1.8.0 

func (c *Catalog) DeleteRepositoryMetadata(ctx context.Context, repository string, keys []string) error

DeleteRepositoryMetadata delete repository metadata

func (*Catalog) DeleteTag 

func (c *Catalog) DeleteTag(ctx context.Context, repositoryID string, tagID string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) Diff 

func (c *Catalog) Diff(ctx context.Context, repositoryID string, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)

func (*Catalog) DiffUncommitted 

func (c *Catalog) DiffUncommitted(ctx context.Context, repositoryID, branch, prefix, delimiter string, limit int, after string) (Differences, bool, error)

func (*Catalog) DumpBranches 

func (c *Catalog) DumpBranches(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) DumpCommits 

func (c *Catalog) DumpCommits(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) DumpRepositoryStatus added in v1.3.0 

func (c *Catalog) DumpRepositoryStatus(ctx context.Context, repositoryID string, id string) (*RepositoryDumpStatus, error)

func (*Catalog) DumpRepositorySubmit added in v1.3.0 

func (c *Catalog) DumpRepositorySubmit(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) DumpTags 

func (c *Catalog) DumpTags(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) FindMergeBase added in v0.94.1 

func (c *Catalog) FindMergeBase(ctx context.Context, repositoryID string, destinationRef string, sourceRef string) (string, string, string, error)

func (*Catalog) GetBranchProtectionRules added in v0.52.0 

func (c *Catalog) GetBranchProtectionRules(ctx context.Context, repositoryID string) (*graveler.BranchProtectionRules, *string, error)

func (*Catalog) GetBranchReference 

func (c *Catalog) GetBranchReference(ctx context.Context, repositoryID string, branch string) (string, error)

func (*Catalog) GetCommit 

func (c *Catalog) GetCommit(ctx context.Context, repositoryID string, reference string) (*CommitLog, error)

func (*Catalog) GetEntry 

func (c *Catalog) GetEntry(ctx context.Context, repositoryID string, reference string, path string, params GetEntryParams) (*DBEntry, error)

GetEntry returns the current entry for a path in repository branch reference. Returns the entry with ExpiredError if it has expired from underlying storage.

func (*Catalog) GetGarbageCollectionRules added in v0.43.0 

func (c *Catalog) GetGarbageCollectionRules(ctx context.Context, repositoryID string) (*graveler.GarbageCollectionRules, error)

func (*Catalog) GetImportStatus added in v0.102.0 

func (c *Catalog) GetImportStatus(ctx context.Context, repositoryID, importID string) (*graveler.ImportStatus, error)

func (*Catalog) GetMetaRange 

func (c *Catalog) GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeAddress, error)

func (*Catalog) GetRange 

func (c *Catalog) GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeAddress, error)

func (*Catalog) GetRepository 

func (c *Catalog) GetRepository(ctx context.Context, repository string) (*Repository, error)

GetRepository get repository information

func (*Catalog) GetRepositoryMetadata added in v0.104.0 

func (c *Catalog) GetRepositoryMetadata(ctx context.Context, repository string) (graveler.RepositoryMetadata, error)

GetRepositoryMetadata get repository metadata

func (*Catalog) GetStagingToken added in v0.40.0 

func (c *Catalog) GetStagingToken(ctx context.Context, repositoryID string, branch string) (*string, error)

func (*Catalog) GetTag 

func (c *Catalog) GetTag(ctx context.Context, repositoryID string, tagID string) (string, error)

func (*Catalog) HardResetBranch added in v1.8.0 

func (c *Catalog) HardResetBranch(ctx context.Context, repositoryID, branch, refExpr string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) Import added in v0.102.0 

func (c *Catalog) Import(ctx context.Context, repositoryID, branchID string, params ImportRequest) (string, error)

func (*Catalog) ListBranches 

func (c *Catalog) ListBranches(ctx context.Context, repositoryID string, prefix string, limit int, after string) ([]*Branch, bool, error)

func (*Catalog) ListCommits 

func (c *Catalog) ListCommits(ctx context.Context, repositoryID string, branch string, params LogParams) ([]*CommitLog, bool, error)

func (*Catalog) ListEntries 

func (c *Catalog) ListEntries(ctx context.Context, repositoryID string, reference string, prefix string, after string, delimiter string, limit int) ([]*DBEntry, bool, error)

func (*Catalog) ListRepositories 

func (c *Catalog) ListRepositories(ctx context.Context, limit int, prefix, after string) ([]*Repository, bool, error)

ListRepositories list repository information, the bool returned is true when more repositories can be listed. In this case, pass the last repository name as 'after' on the next call to ListRepositories

func (*Catalog) ListTags 

func (c *Catalog) ListTags(ctx context.Context, repositoryID string, prefix string, limit int, after string) ([]*Tag, bool, error)

func (*Catalog) LoadBranches 

func (c *Catalog) LoadBranches(ctx context.Context, repositoryID, branchesMetaRangeID string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) LoadCommits 

func (c *Catalog) LoadCommits(ctx context.Context, repositoryID, commitsMetaRangeID string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) LoadTags 

func (c *Catalog) LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) Merge 

func (c *Catalog) Merge(ctx context.Context, repositoryID string, destinationBranch string, sourceRef string, committer string, message string, metadata Metadata, strategy string, opts ...graveler.SetOptionsFunc) (string, error)

func (*Catalog) PrepareExpiredCommits added in v0.43.0 

func (c *Catalog) PrepareExpiredCommits(ctx context.Context, repositoryID string) (*graveler.GarbageCollectionRunMetadata, error)

func (*Catalog) PrepareGCUncommitted added in v0.85.0 

func (c *Catalog) PrepareGCUncommitted(ctx context.Context, repositoryID string, mark *GCUncommittedMark) (*PrepareGCUncommittedInfo, error)

func (*Catalog) ResetBranch 

func (c *Catalog) ResetBranch(ctx context.Context, repositoryID string, branch string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) ResetEntries 

func (c *Catalog) ResetEntries(ctx context.Context, repositoryID string, branch string, prefix string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) ResetEntry 

func (c *Catalog) ResetEntry(ctx context.Context, repositoryID string, branch string, path string, opts ...graveler.SetOptionsFunc) error

func (*Catalog) RestoreRepositoryStatus added in v1.3.0 

func (c *Catalog) RestoreRepositoryStatus(ctx context.Context, repositoryID string, id string) (*RepositoryRestoreStatus, error)

func (*Catalog) RestoreRepositorySubmit added in v1.3.0 

func (c *Catalog) RestoreRepositorySubmit(ctx context.Context, repositoryID string, info *RepositoryDumpInfo, opts ...graveler.SetOptionsFunc) (string, error)

func (*Catalog) Revert 

func (c *Catalog) Revert(ctx context.Context, repositoryID string, branch string, params RevertParams, opts ...graveler.SetOptionsFunc) error

func (*Catalog) SetBranchProtectionRules added in v0.111.0 

func (c *Catalog) SetBranchProtectionRules(ctx context.Context, repositoryID string, rules *graveler.BranchProtectionRules, lastKnownChecksum *string) error

func (*Catalog) SetGarbageCollectionRules added in v0.43.0 

func (c *Catalog) SetGarbageCollectionRules(ctx context.Context, repositoryID string, rules *graveler.GarbageCollectionRules) error

func (*Catalog) SetHooksHandler 

func (c *Catalog) SetHooksHandler(hooks graveler.HooksHandler)

func (*Catalog) SetLinkAddress added in v0.90.0 

func (c *Catalog) SetLinkAddress(ctx context.Context, repository, physicalAddress string) error

SetLinkAddress to validate single use limited in time of a given physical address

func (*Catalog) UpdateRepositoryMetadata added in v1.8.0 

func (c *Catalog) UpdateRepositoryMetadata(ctx context.Context, repository string, metadata graveler.RepositoryMetadata) error

UpdateRepositoryMetadata set repository metadata

func (*Catalog) VerifyLinkAddress added in v0.90.0 

func (c *Catalog) VerifyLinkAddress(ctx context.Context, repository, physicalAddress string) error

func (*Catalog) WriteMetaRange added in v0.65.0 

func (c *Catalog) WriteMetaRange(ctx context.Context, repositoryID string, ranges []*graveler.RangeInfo, opts ...graveler.SetOptionsFunc) (*graveler.MetaRangeInfo, error)

func (*Catalog) WriteRange added in v0.65.0 

func (c *Catalog) WriteRange(ctx context.Context, repositoryID string, params WriteRangeRequest, opts ...graveler.SetOptionsFunc) (*graveler.RangeInfo, *Mark, error)

type CherryPickParams added in v0.97.0 

type CherryPickParams struct {
	Reference    string // the commit to pick
	ParentNumber *int   // if a merge commit was picked, the change will be applied relative to this parent number (1-based).
	Committer    string
}

type CommitGeneration added in v1.9.0 

type CommitGeneration int64

type CommitLog 

type CommitLog struct {
	Reference    string
	Committer    string
	Message      string
	CreationDate time.Time
	Metadata     Metadata
	MetaRangeID  string
	Parents      []string
	Generation   CommitGeneration
	Version      CommitVersion
}

func CommitRecordToLog added in v0.102.0 

func CommitRecordToLog(val *graveler.CommitRecord) *CommitLog

type CommitVersion added in v1.9.0 

type CommitVersion int

type Config 

type Config struct {
	Config                *config.Config
	KVStore               kv.Store
	WalkerFactory         WalkerFactory
	SettingsManagerOption settings.ManagerOption
	PathProvider          *upload.PathPartitionProvider
}

type DBEntry 

type DBEntry struct {
	CommonLevel     bool
	Path            string
	PhysicalAddress string
	CreationDate    time.Time
	Size            int64
	Checksum        string
	Metadata        Metadata
	Expired         bool
	AddressType     AddressType
	ContentType     string
}

type DBEntryBuilder added in v0.53.0 

type DBEntryBuilder struct {
	// contains filtered or unexported fields
}

DBEntryBuilder DBEntry builder

func NewDBEntryBuilder added in v0.53.0 

func NewDBEntryBuilder() *DBEntryBuilder

func (*DBEntryBuilder) AddressType added in v0.53.0 

func (b *DBEntryBuilder) AddressType(addressType AddressType) *DBEntryBuilder

func (*DBEntryBuilder) Build added in v0.53.0 

func (b *DBEntryBuilder) Build() DBEntry

func (*DBEntryBuilder) Checksum added in v0.53.0 

func (b *DBEntryBuilder) Checksum(checksum string) *DBEntryBuilder

func (*DBEntryBuilder) CommonLevel added in v0.53.0 

func (b *DBEntryBuilder) CommonLevel(commonLevel bool) *DBEntryBuilder

func (*DBEntryBuilder) ContentType added in v0.53.0 

func (b *DBEntryBuilder) ContentType(contentType string) *DBEntryBuilder

func (*DBEntryBuilder) CreationDate added in v0.53.0 

func (b *DBEntryBuilder) CreationDate(creationDate time.Time) *DBEntryBuilder

func (*DBEntryBuilder) Expired added in v0.53.0 

func (b *DBEntryBuilder) Expired(expired bool) *DBEntryBuilder

func (*DBEntryBuilder) Metadata added in v0.53.0 

func (b *DBEntryBuilder) Metadata(metadata Metadata) *DBEntryBuilder

func (*DBEntryBuilder) Path added in v0.53.0 

func (b *DBEntryBuilder) Path(path string) *DBEntryBuilder

func (*DBEntryBuilder) PhysicalAddress added in v0.53.0 

func (b *DBEntryBuilder) PhysicalAddress(physicalAddress string) *DBEntryBuilder

func (*DBEntryBuilder) RelativeAddress added in v0.53.0 

func (b *DBEntryBuilder) RelativeAddress(relative bool) *DBEntryBuilder

func (*DBEntryBuilder) Size added in v0.53.0 

func (b *DBEntryBuilder) Size(size int64) *DBEntryBuilder

type DiffParams 

type DiffParams struct {
	Limit            int
	After            string
	Prefix           string
	Delimiter        string
	AdditionalFields []string // db fields names that will be load in additional to Path on Difference's Entry
}

type DiffResultRecord 

type DiffResultRecord struct {
	TargetEntryNotInDirectBranch bool // the entry is reflected via lineage, NOT in the branch itself
	Difference
}

type Difference 

type Difference struct {
	DBEntry                // Partially filled. Path is always set.
	Type    DifferenceType `db:"diff_type"`
}

func (Difference) String 

func (d Difference) String() string

type DifferenceType 

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypePrefixChanged
	DifferenceTypeConflict
	DifferenceTypeNone
)

type Differences 

type Differences []Difference

func (Differences) Equal 

func (d Differences) Equal(other Differences) bool

type Entry 

type Entry struct {
	Address      string                 `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
	LastModified *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_modified,json=lastModified,proto3" json:"last_modified,omitempty"`
	Size         int64                  `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
	ETag         string                 `protobuf:"bytes,4,opt,name=e_tag,json=eTag,proto3" json:"e_tag,omitempty"`
	Metadata     map[string]string      `` /* 157-byte string literal not displayed */
	AddressType  Entry_AddressType      `` /* 126-byte string literal not displayed */
	ContentType  string                 `protobuf:"bytes,7,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"`
	// contains filtered or unexported fields
}

func ValueToEntry 

func ValueToEntry(value *graveler.Value) (*Entry, error)

func (*Entry) Descriptor deprecated 

func (*Entry) Descriptor() ([]byte, []int)

Deprecated: Use Entry.ProtoReflect.Descriptor instead.

func (*Entry) GetAddress 

func (x *Entry) GetAddress() string

func (*Entry) GetAddressType added in v0.40.0 

func (x *Entry) GetAddressType() Entry_AddressType

func (*Entry) GetContentType added in v0.53.0 

func (x *Entry) GetContentType() string

func (*Entry) GetETag 

func (x *Entry) GetETag() string

func (*Entry) GetLastModified 

func (x *Entry) GetLastModified() *timestamppb.Timestamp

func (*Entry) GetMetadata 

func (x *Entry) GetMetadata() map[string]string

func (*Entry) GetSize 

func (x *Entry) GetSize() int64

func (*Entry) ProtoMessage 

func (*Entry) ProtoMessage()

func (*Entry) ProtoReflect 

func (x *Entry) ProtoReflect() protoreflect.Message

func (*Entry) Reset 

func (x *Entry) Reset()

func (*Entry) String 

func (x *Entry) String() string

type EntryDiff 

type EntryDiff struct {
	Type  graveler.DiffType
	Path  Path
	Entry *Entry
}

type EntryDiffIterator 

type EntryDiffIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryDiff
	Err() error
	Close()
}

func NewEntryDiffIterator 

func NewEntryDiffIterator(it graveler.DiffIterator) EntryDiffIterator

type EntryIterator 

type EntryIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryRecord
	Err() error
	Close()
}

func NewPrefixIterator 

func NewPrefixIterator(it EntryIterator, prefix Path) EntryIterator

type EntryListing 

type EntryListing struct {
	CommonPrefix bool
	Path
	*Entry
}

type EntryListingIterator 

type EntryListingIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryListing
	Err() error
	Close()
}

func NewEntryListingIterator 

func NewEntryListingIterator(it EntryIterator, prefix Path, delimiter Path) EntryListingIterator

type EntryRecord 

type EntryRecord struct {
	Path Path
	*Entry
}

type EntryWithMarker added in v0.65.0 

type EntryWithMarker struct {
	EntryRecord
	Mark
}

type Entry_AddressType added in v0.40.0 

type Entry_AddressType int32
const (
	// Deprecated.
	// Unknown address type (should only exist for old commits)
	// is resolved (to Relative or Full) by the prefix of the address.
	Entry_BY_PREFIX_DEPRECATED Entry_AddressType = 0
	Entry_RELATIVE             Entry_AddressType = 1
	Entry_FULL                 Entry_AddressType = 2
)

func (Entry_AddressType) Descriptor added in v0.40.0 

func (Entry_AddressType) Descriptor() protoreflect.EnumDescriptor

func (Entry_AddressType) Enum added in v0.40.0 

func (x Entry_AddressType) Enum() *Entry_AddressType

func (Entry_AddressType) EnumDescriptor deprecated added in v0.40.0 

func (Entry_AddressType) EnumDescriptor() ([]byte, []int)

Deprecated: Use Entry_AddressType.Descriptor instead.

func (Entry_AddressType) Number added in v0.40.0 

func (x Entry_AddressType) Number() protoreflect.EnumNumber

func (Entry_AddressType) String added in v0.40.0 

func (x Entry_AddressType) String() string

func (Entry_AddressType) Type added in v0.40.0 

func (Entry_AddressType) Type() protoreflect.EnumType

type ExpireResult 

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows 

type ExpiryRows interface {
	Close()
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type GCUncommittedMark added in v0.85.0 

type GCUncommittedMark struct {
	BranchID graveler.BranchID `json:"branch"`
	Path     Path              `json:"path"`
	RunID    string            `json:"run_id"`
	Key      string            `json:"key"`
}

GCUncommittedMark Marks the *next* item to be scanned by the paginated call to PrepareGCUncommitted

type GetEntryParams 

type GetEntryParams struct {
	// StageOnly when true will return entry found on stage without checking committed data
	StageOnly bool
}

GetEntryParams configures what entries GetEntry returns.

type Import added in v0.102.0 

type Import struct {
	// contains filtered or unexported fields
}

func NewImport added in v0.102.0 

func NewImport(ctx context.Context, cancel context.CancelFunc, logger logging.Logger, kvStore kv.Store, repository *graveler.RepositoryRecord, importID string) (*Import, error)

func (*Import) Close added in v0.102.0 

func (i *Import) Close()

func (*Import) Closed added in v0.102.0 

func (i *Import) Closed() bool

func (*Import) Ingest added in v0.102.0 

func (i *Import) Ingest(it *walkEntryIterator) error

func (*Import) NewItr added in v0.102.0 

func (i *Import) NewItr() (*importIterator, error)

func (*Import) SetError added in v0.102.0 

func (i *Import) SetError(err error)

func (*Import) SetStatus added in v0.102.0 

func (i *Import) SetStatus(status graveler.ImportStatus)

func (*Import) Status added in v0.102.0 

func (i *Import) Status() graveler.ImportStatus

type ImportCommit added in v0.102.0 

type ImportCommit struct {
	CommitMessage string
	Committer     string
	Metadata      Metadata
}

type ImportPath added in v0.102.0 

type ImportPath struct {
	Path        string
	Destination string
	Type        ImportPathType
}

type ImportPathType added in v0.102.0 

type ImportPathType string

func GetImportPathType added in v0.102.0 

func GetImportPathType(t string) (ImportPathType, error)

type ImportRequest added in v0.102.0 

type ImportRequest struct {
	Paths  []ImportPath
	Commit ImportCommit
	Force  bool
}

type LogParams added in v0.54.0 

type LogParams struct {
	PathList      []PathRecord
	FromReference string
	Amount        int
	Limit         bool
	FirstParent   bool
	Since         *time.Time
	StopAt        string
}

type Mark added in v0.65.0 

type Mark struct {
	block.Mark
	StagingToken string
}

Mark stands for pagination information when listing objects from the blockstore. It is used for server-client communication on the status of range ingestion.

type Metadata 

type Metadata map[string]string

func (*Metadata) Scan 

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value 

func (j Metadata) Value() (driver.Value, error)

type Path 

type Path string

func (Path) String 

func (id Path) String() string

type PathRecord added in v0.54.0 

type PathRecord struct {
	Path     Path
	IsPrefix bool
}

type PrepareGCUncommittedInfo added in v0.85.0 

type PrepareGCUncommittedInfo struct {
	RunID    string `json:"run_id"`
	Location string `json:"location"`
	Filename string `json:"filename"`
	Mark     *GCUncommittedMark
}

type Repository 

type Repository struct {
	Name             string
	StorageNamespace string
	DefaultBranch    string
	CreationDate     time.Time
	ReadOnly         bool
}

type RepositoryDumpInfo added in v1.3.0 

type RepositoryDumpInfo struct {
	CommitsMetarangeId  string `protobuf:"bytes,1,opt,name=commits_metarange_id,json=commitsMetarangeId,proto3" json:"commits_metarange_id,omitempty"`
	TagsMetarangeId     string `protobuf:"bytes,2,opt,name=tags_metarange_id,json=tagsMetarangeId,proto3" json:"tags_metarange_id,omitempty"`
	BranchesMetarangeId string `protobuf:"bytes,3,opt,name=branches_metarange_id,json=branchesMetarangeId,proto3" json:"branches_metarange_id,omitempty"`
	// contains filtered or unexported fields
}

RepositoryDumpInfo holds the metarange IDs for a repository dump

func (*RepositoryDumpInfo) Descriptor deprecated added in v1.3.0 

func (*RepositoryDumpInfo) Descriptor() ([]byte, []int)

Deprecated: Use RepositoryDumpInfo.ProtoReflect.Descriptor instead.

func (*RepositoryDumpInfo) GetBranchesMetarangeId added in v1.3.0 

func (x *RepositoryDumpInfo) GetBranchesMetarangeId() string

func (*RepositoryDumpInfo) GetCommitsMetarangeId added in v1.3.0 

func (x *RepositoryDumpInfo) GetCommitsMetarangeId() string

func (*RepositoryDumpInfo) GetTagsMetarangeId added in v1.3.0 

func (x *RepositoryDumpInfo) GetTagsMetarangeId() string

func (*RepositoryDumpInfo) ProtoMessage added in v1.3.0 

func (*RepositoryDumpInfo) ProtoMessage()

func (*RepositoryDumpInfo) ProtoReflect added in v1.3.0 

func (x *RepositoryDumpInfo) ProtoReflect() protoreflect.Message

func (*RepositoryDumpInfo) Reset added in v1.3.0 

func (x *RepositoryDumpInfo) Reset()

func (*RepositoryDumpInfo) String added in v1.3.0 

func (x *RepositoryDumpInfo) String() string

type RepositoryDumpStatus added in v1.3.0 

type RepositoryDumpStatus struct {
	Task *Task               `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"`
	Info *RepositoryDumpInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"`
	// contains filtered or unexported fields
}

RepositoryDumpStatus holds the status of a repository dump

func (*RepositoryDumpStatus) Descriptor deprecated added in v1.3.0 

func (*RepositoryDumpStatus) Descriptor() ([]byte, []int)

Deprecated: Use RepositoryDumpStatus.ProtoReflect.Descriptor instead.

func (*RepositoryDumpStatus) GetInfo added in v1.3.0 

func (x *RepositoryDumpStatus) GetInfo() *RepositoryDumpInfo

func (*RepositoryDumpStatus) GetTask added in v1.3.0 

func (x *RepositoryDumpStatus) GetTask() *Task

func (*RepositoryDumpStatus) ProtoMessage added in v1.3.0 

func (*RepositoryDumpStatus) ProtoMessage()

func (*RepositoryDumpStatus) ProtoReflect added in v1.3.0 

func (x *RepositoryDumpStatus) ProtoReflect() protoreflect.Message

func (*RepositoryDumpStatus) Reset added in v1.3.0 

func (x *RepositoryDumpStatus) Reset()

func (*RepositoryDumpStatus) String added in v1.3.0 

func (x *RepositoryDumpStatus) String() string

type RepositoryRestoreStatus added in v1.3.0 

type RepositoryRestoreStatus struct {
	Task *Task `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"`
	// contains filtered or unexported fields
}

RepositoryRestoreStatus holds the status of a repository restore

func (*RepositoryRestoreStatus) Descriptor deprecated added in v1.3.0 

func (*RepositoryRestoreStatus) Descriptor() ([]byte, []int)

Deprecated: Use RepositoryRestoreStatus.ProtoReflect.Descriptor instead.

func (*RepositoryRestoreStatus) GetTask added in v1.3.0 

func (x *RepositoryRestoreStatus) GetTask() *Task

func (*RepositoryRestoreStatus) ProtoMessage added in v1.3.0 

func (*RepositoryRestoreStatus) ProtoMessage()

func (*RepositoryRestoreStatus) ProtoReflect added in v1.3.0 

func (x *RepositoryRestoreStatus) ProtoReflect() protoreflect.Message

func (*RepositoryRestoreStatus) Reset added in v1.3.0 

func (x *RepositoryRestoreStatus) Reset()

func (*RepositoryRestoreStatus) String added in v1.3.0 

func (x *RepositoryRestoreStatus) String() string

type RevertParams 

type RevertParams struct {
	Reference    string // the commit to revert
	ParentNumber int    // if reverting a merge commit, the change will be reversed relative to this parent number (1-based).
	Committer    string
	AllowEmpty   bool // allow empty commit (revert without changes)
}

type Store 

type Store interface {
	graveler.KeyValueStore
	graveler.VersionController
	graveler.Dumper
	graveler.Loader
	graveler.Plumbing
}

type Tag 

type Tag struct {
	ID       string
	CommitID string
}

type Task added in v1.3.0 

type Task struct {
	Id        string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	Done      bool                   `protobuf:"varint,2,opt,name=done,proto3" json:"done,omitempty"`
	UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"`
	Progress  int64                  `protobuf:"varint,4,opt,name=progress,proto3" json:"progress,omitempty"`
	Error     string                 `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"`
	// contains filtered or unexported fields
}

Task is a generic task status message

func (*Task) Descriptor deprecated added in v1.3.0 

func (*Task) Descriptor() ([]byte, []int)

Deprecated: Use Task.ProtoReflect.Descriptor instead.

func (*Task) GetDone added in v1.3.0 

func (x *Task) GetDone() bool

func (*Task) GetError added in v1.3.0 

func (x *Task) GetError() string

func (*Task) GetId added in v1.3.0 

func (x *Task) GetId() string

func (*Task) GetProgress added in v1.3.0 

func (x *Task) GetProgress() int64

func (*Task) GetUpdatedAt added in v1.3.0 

func (x *Task) GetUpdatedAt() *timestamppb.Timestamp

func (*Task) ProtoMessage added in v1.3.0 

func (*Task) ProtoMessage()

func (*Task) ProtoReflect added in v1.3.0 

func (x *Task) ProtoReflect() protoreflect.Message

func (*Task) Reset added in v1.3.0 

func (x *Task) Reset()

func (*Task) String added in v1.3.0 

func (x *Task) String() string

type TaskMsg added in v1.3.0 

type TaskMsg struct {
	Task *Task `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"`
	// contains filtered or unexported fields
}

TaskMsg described generic message with Task field used for all status messages and for cleanup messages

func (*TaskMsg) Descriptor deprecated added in v1.3.0 

func (*TaskMsg) Descriptor() ([]byte, []int)

Deprecated: Use TaskMsg.ProtoReflect.Descriptor instead.

func (*TaskMsg) GetTask added in v1.3.0 

func (x *TaskMsg) GetTask() *Task

func (*TaskMsg) ProtoMessage added in v1.3.0 

func (*TaskMsg) ProtoMessage()

func (*TaskMsg) ProtoReflect added in v1.3.0 

func (x *TaskMsg) ProtoReflect() protoreflect.Message

func (*TaskMsg) Reset added in v1.3.0 

func (x *TaskMsg) Reset()

func (*TaskMsg) String added in v1.3.0 

func (x *TaskMsg) String() string

type UncommittedIterator added in v0.85.0 

type UncommittedIterator struct {
	// contains filtered or unexported fields
}

func NewUncommittedIterator added in v0.85.0 

func NewUncommittedIterator(ctx context.Context, store Store, repository *graveler.RepositoryRecord) (*UncommittedIterator, error)

func (*UncommittedIterator) Close added in v0.85.0 

func (u *UncommittedIterator) Close()

func (*UncommittedIterator) Err added in v0.85.0 

func (u *UncommittedIterator) Err() error

func (*UncommittedIterator) Next added in v0.85.0 

func (u *UncommittedIterator) Next() bool

Next returns the next entry - if entryItr is still valid - gets the next value from it otherwise call u.next

func (*UncommittedIterator) SeekGE added in v0.85.0 

func (u *UncommittedIterator) SeekGE(branchID graveler.BranchID, id Path)

func (*UncommittedIterator) Value added in v0.85.0 

func (u *UncommittedIterator) Value() *UncommittedRecord

type UncommittedParquetObject added in v0.85.0 

type UncommittedParquetObject struct {
	PhysicalAddress string `parquet:"name=physical_address, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`
	CreationDate    int64  `parquet:"name=creation_date, type=INT64, convertedtype=INT_64"`
}

type UncommittedRecord added in v0.85.0 

type UncommittedRecord struct {
	*EntryRecord
	// contains filtered or unexported fields
}

type UncommittedWriter added in v0.85.0 

type UncommittedWriter struct {
	// contains filtered or unexported fields
}

UncommittedWriter wraps io.Writer and tracks the total size of writes done on this writer Used to get the current file size written without expensive calls to Flush and Stat

func NewUncommittedWriter added in v0.85.0 

func NewUncommittedWriter(writer io.Writer) *UncommittedWriter

func (*UncommittedWriter) Size added in v0.85.0 

func (w *UncommittedWriter) Size() int64

func (*UncommittedWriter) Write added in v0.85.0 

func (w *UncommittedWriter) Write(p []byte) (n int, err error)

type WalkerFactory added in v0.65.0 

type WalkerFactory interface {
	GetWalker(ctx context.Context, opts store.WalkerOptions) (*store.WalkerWrapper, error)
}

WalkerFactory provides an abstraction for creating Walker

type WriteRangeRequest added in v0.98.0 

type WriteRangeRequest struct {
	SourceURI         string
	Prepend           string
	After             string
	StagingToken      string
	ContinuationToken string
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.