datas

package
v0.40.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 19, 2022 License: Apache-2.0 Imports: 18 Imported by: 3

Documentation

Overview

Package datas defines and implements the database layer used in Noms.

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrOptimisticLockFailed = errors.New("optimistic lock failed on database Root update")
	ErrMergeNeeded          = errors.New("dataset head is not ancestor of commit")
	ErrAlreadyCommitted     = errors.New("dataset head already pointing at given commit")
)
View Source 
var CommitLoc = time.Local
View Source 
var CommitNowFunc = time.Now
View Source 
var DatasetFullRe = regexp.MustCompile("^" + DatasetRe.String() + "$")

DatasetFullRe is a regexp that matches a only a target string that is entirely legal Dataset name.

View Source 
var DatasetRe = regexp.MustCompile(`[a-zA-Z0-9\-_/]+`)

DatasetRe is a regexp that matches a legal Dataset name anywhere within the target string.

View Source 
var ErrEmailNotConfigured = errors.New("Aborting commit due to empty committer email. Is your config set?")
View Source 
var ErrEmptyCommitMessage = errors.New("Aborting commit due to empty commit message.")
View Source 
var ErrInvalidDatasetID = errors.New("Invalid dataset ID")
View Source 
var ErrNameNotConfigured = errors.New("Aborting commit due to empty committer name. Is your config set?")
View Source 
var TagLoc = CommitLoc
View Source 
var TagNowFunc = CommitNowFunc

Functions

func CanUsePuller 

func CanUsePuller(db Database) bool

CanUsePuller returns true if a datas.Puller can be used to pull data from one Database into another. Not all Databases support this yet.

func ChunkStoreFromDatabase 

func ChunkStoreFromDatabase(db Database) chunks.ChunkStore

func FindClosureCommonAncestor 

func FindClosureCommonAncestor(ctx context.Context, cl CommitClosure, cm *Commit, vr types.ValueReader) (a hash.Hash, ok bool, err error)

FindClosureCommonAncestor returns the most recent common ancestor of |cl| and |cm|, where |cl| is the transitive closure of one or more refs. If a common ancestor exists, |ok| is set to true, else false.

func FindCommonAncestor 

func FindCommonAncestor(ctx context.Context, c1, c2 *Commit, vr1, vr2 types.ValueReader) (hash.Hash, bool, error)

FindCommonAncestor returns the most recent common ancestor of c1 and c2, if one exists, setting ok to true. If there is no common ancestor, ok is set to false. Refs of |c1| are dereferenced through |vr1|, while refs of |c2| are dereference through |vr2|.

This implementation makes use of the parents_closure field on the commit struct. If the commit does not have a materialized parents_closure, this implementation delegates to findCommonAncestorUsingParentsList.

func GetCSStatSummaryForDB 

func GetCSStatSummaryForDB(db Database) string

func GetCommittedValue 

func GetCommittedValue(ctx context.Context, vr types.ValueReader, cv types.Value) (types.Value, error)

func IsCommit 

func IsCommit(v types.Value) (bool, error)

func IsCommitType 

func IsCommitType(nbf *types.NomsBinFormat, t *types.Type) bool

func IsTag 

func IsTag(v types.Value) (bool, error)

func IsValidDatasetName 

func IsValidDatasetName(name string) bool

func IsWorkingSet 

func IsWorkingSet(v types.Value) (bool, error)

func PruneTableFiles 

func PruneTableFiles(ctx context.Context, db Database) error

func RefMapApplyEdits 

func RefMapApplyEdits(rm *serial.RefMap, builder *flatbuffers.Builder, edits []RefMapEdit) flatbuffers.UOffsetT

func RefMapLookup 

func RefMapLookup(rm *serial.RefMap, key string) hash.Hash

Types

type Commit 

type Commit struct {
	// contains filtered or unexported fields
}

func GetCommitParents 

func GetCommitParents(ctx context.Context, vr types.ValueReader, cv types.Value) ([]*Commit, error)

GetCommitParents returns |Ref|s to the parents of the commit.

func LoadCommitAddr 

func LoadCommitAddr(ctx context.Context, vr types.ValueReader, addr hash.Hash) (*Commit, error)

func LoadCommitRef 

func LoadCommitRef(ctx context.Context, vr types.ValueReader, r types.Ref) (*Commit, error)

func NewCommitForValue 

func NewCommitForValue(ctx context.Context, vrw types.ValueReadWriter, v types.Value, opts CommitOptions) (*Commit, error)

func (*Commit) Addr 

func (c *Commit) Addr() hash.Hash

func (*Commit) Height 

func (c *Commit) Height() uint64

func (*Commit) NomsValue 

func (c *Commit) NomsValue() types.Value

type CommitByHeightHeap 

type CommitByHeightHeap []*Commit

func (CommitByHeightHeap) Empty 

func (r CommitByHeightHeap) Empty() bool

func (CommitByHeightHeap) Len 

func (r CommitByHeightHeap) Len() int

func (CommitByHeightHeap) Less 

func (r CommitByHeightHeap) Less(i, j int) bool

func (CommitByHeightHeap) MaxHeight 

func (r CommitByHeightHeap) MaxHeight() uint64

func (*CommitByHeightHeap) Pop 

func (r *CommitByHeightHeap) Pop() interface{}

func (*CommitByHeightHeap) PopCommitsOfHeight 

func (r *CommitByHeightHeap) PopCommitsOfHeight(h uint64) []*Commit

func (*CommitByHeightHeap) Push 

func (r *CommitByHeightHeap) Push(x interface{})

func (CommitByHeightHeap) Swap 

func (r CommitByHeightHeap) Swap(i, j int)

type CommitClosure 

type CommitClosure interface {
	// Contains returns true if |commit| is contained in the closure.
	Contains(ctx context.Context, commit *Commit) (bool, error)
}

CommitClosure is a transitive closure of commit parents.

func NewLazyCommitClosure 

func NewLazyCommitClosure(commit *Commit, vr types.ValueReader) CommitClosure

NewLazyCommitClosure makes a lazy CommitClosure, which computes the transitive closure of |commit| on demand to answer Contains() queries.

func NewSetCommitClosure 

func NewSetCommitClosure(ctx context.Context, vr types.ValueReader, commit *Commit) (CommitClosure, error)

NewSetCommitClosure computes the entire transitive closure of |commit|.

type CommitMeta 

type CommitMeta struct {
	Name          string
	Email         string
	Timestamp     uint64
	Description   string
	UserTimestamp int64
}

CommitMeta contains all the metadata that is associated with a commit within a data repo.

func CommitMetaFromNomsSt 

func CommitMetaFromNomsSt(st types.Struct) (*CommitMeta, error)

func GetCommitMeta 

func GetCommitMeta(ctx context.Context, cv types.Value) (*CommitMeta, error)

GetCommitMeta extracts the CommitMeta field from a commit. Returns |nil, nil| if there is no metadata for the commit.

func NewCommitMeta 

func NewCommitMeta(name, email, desc string) (*CommitMeta, error)

NewCommitMeta creates a CommitMeta instance from a name, email, and description and uses the current time for the timestamp

func NewCommitMetaWithUserTS 

func NewCommitMetaWithUserTS(name, email, desc string, userTS time.Time) (*CommitMeta, error)

NewCommitMetaWithUserTS creates a user metadata

func (*CommitMeta) FormatTS 

func (cm *CommitMeta) FormatTS() string

FormatTS takes the internal timestamp and turns it into a human readable string in the time.RubyDate format which looks like: "Mon Jan 02 15:04:05 -0700 2006"

func (*CommitMeta) String 

func (cm *CommitMeta) String() string

String returns the human readable string representation of the commit data

func (*CommitMeta) Time 

func (cm *CommitMeta) Time() time.Time

Time returns the time at which the commit occurred

type CommitOptions 

type CommitOptions struct {
	// Parents, if provided, is the parent commits of the commit we are
	// creating. If it is empty, the existing dataset head will be the only
	// parent.
	Parents []hash.Hash

	Meta *CommitMeta
}

CommitOptions is used to pass options into Commit.

type Database 

type Database interface {
	// Close must have no side-effects
	io.Closer

	// Datasets returns the root of the database which is a
	// Map<String, Ref<Commit>> where string is a datasetID.
	Datasets(ctx context.Context) (DatasetsMap, error)

	// GetDataset returns a Dataset struct containing the current mapping of
	// datasetID in the above Datasets Map.
	GetDataset(ctx context.Context, datasetID string) (Dataset, error)

	// Commit updates the Commit that ds.ID() in this database points at. All
	// Values that have been written to this Database are guaranteed to be
	// persistent after Commit() returns successfully.
	//
	// The new Commit struct is constructed using v, opts.ParentsList, and
	// opts.Meta. If opts.ParentsList is empty then the head value from
	// |ds| is used as the parent. If opts.Meta is the zero value the
	// Commit struct as an empty Meta struct.
	//
	// If the update cannot be performed because the existing dataset head
	// is not a common ancestor of the constructed commit struct, returns
	// an 'ErrMergeNeeded' error.
	Commit(ctx context.Context, ds Dataset, v types.Value, opts CommitOptions) (Dataset, error)

	// Tag stores an immutable reference to a Commit. It takes a Hash to
	// the Commit and a Dataset whose head must be nil (ie a newly created
	// Dataset).  The new Tag struct is constructed pointing at
	// |commitAddr| and metadata about the tag contained in the struct
	// `opts.Meta`.
	Tag(ctx context.Context, ds Dataset, commitAddr hash.Hash, opts TagOptions) (Dataset, error)

	// UpdateWorkingSet updates the dataset given, setting its value to a new
	// working set value object with the ref and meta given. If the dataset given
	// already had a value, it must match the hash given or this method returns
	// ErrOptimisticLockFailed and the caller must retry.
	// The returned Dataset is always the newest snapshot, regardless of
	// success or failure, and Datasets() is updated to match backing storage
	// upon return as well.
	UpdateWorkingSet(ctx context.Context, ds Dataset, workingSet WorkingSetSpec, prevHash hash.Hash) (Dataset, error)

	// CommitWithWorkingSet combines Commit and UpdateWorkingSet, combining the parameters of both. It uses the
	// pessimistic lock that UpdateWorkingSet does, asserting that the hash |prevWsHash| given is still the current one
	// before attempting to write a new value. And it does the normal optimistic locking that Commit does, assuming the
	// pessimistic locking passes. After this method runs, the two datasets given in |commitDS and |workingSetDS| are both
	// updated in the new root, or neither of them are.
	CommitWithWorkingSet(ctx context.Context, commitDS, workingSetDS Dataset, val types.Value, workingSetSpec WorkingSetSpec, prevWsHash hash.Hash, opts CommitOptions) (Dataset, Dataset, error)

	// Delete removes the Dataset named ds.ID() from the map at the root of
	// the Database. If the Dataset is already not present in the map,
	// returns success.
	//
	// If the update cannot be performed, e.g., because of a conflict,
	// Delete returns an 'ErrMergeNeeded' error.
	Delete(ctx context.Context, ds Dataset) (Dataset, error)

	// SetHead ignores any lineage constraints (e.g. the current head being
	// an ancestor of the new Commit) and force-sets a mapping from
	// datasetID: addr in this database. addr can point to a Commit or a
	// Tag, but if Dataset is already present in the Database, it must
	// point to the type of struct.
	//
	// All values that have been written to this Database are guaranteed to
	// be persistent after SetHead(). If the update cannot be performed,
	// error will be non-nil.
	SetHead(ctx context.Context, ds Dataset, newHeadAddr hash.Hash) (Dataset, error)

	// FastForward takes a types.Ref to a Commit object and makes it the new
	// Head of ds iff it is a descendant of the current Head. Intended to be
	// used e.g. after a call to Pull(). If the update cannot be performed,
	// e.g., because another process moved the current Head out from under
	// you, err will be non-nil.
	FastForward(ctx context.Context, ds Dataset, newHeadAddr hash.Hash) (Dataset, error)

	// Stats may return some kind of struct that reports statistics about the
	// ChunkStore that backs this Database instance. The type is
	// implementation-dependent, and impls may return nil
	Stats() interface{}

	// StatsSummary may return a string containing summarized statistics for
	// the ChunkStore that backs this Database. It must return "Unsupported"
	// if this operation is not supported.
	StatsSummary() string
	// contains filtered or unexported methods
}

Database provides versioned storage for noms values. While Values can be directly read and written from a Database, it is generally more appropriate to read data by inspecting the Head of a Dataset and write new data by updating the Head of a Dataset via Commit() or similar. Particularly, new data is not guaranteed to be persistent until after a Commit (Delete, SetHeadToCommit, or FastForward) operation completes. The Database API is stateful, meaning that calls to GetDataset() or Datasets() occurring after a call to Commit() (et al) will represent the result of the Commit().

func NewDatabase 

func NewDatabase(cs chunks.ChunkStore) Database

func NewTypesDatabase 

func NewTypesDatabase(vs *types.ValueStore) Database

type Dataset 

type Dataset struct {
	// contains filtered or unexported fields
}

Dataset is a named value within a Database. Different head values may be stored in a dataset. Most commonly, this is a commit, but other values are also supported in some cases.

func CommitValue 

func CommitValue(ctx context.Context, db Database, ds Dataset, v types.Value) (Dataset, error)

Calls db.Commit with empty CommitOptions{}.

func (Dataset) Database 

func (ds Dataset) Database() Database

Database returns the Database object in which this Dataset is stored. WARNING: This method is under consideration for deprecation.

func (Dataset) HasHead 

func (ds Dataset) HasHead() bool

HasHead() returns 'true' if this dataset has a Head Commit, false otherwise.

func (Dataset) HeadTag 

func (ds Dataset) HeadTag() (*TagMeta, hash.Hash, error)

func (Dataset) HeadWorkingSet 

func (ds Dataset) HeadWorkingSet() (*WorkingSetHead, error)

func (Dataset) ID 

func (ds Dataset) ID() string

ID returns the name of this Dataset.

func (Dataset) IsTag 

func (ds Dataset) IsTag() bool

func (Dataset) IsWorkingSet 

func (ds Dataset) IsWorkingSet() bool

func (Dataset) MaybeHead 

func (ds Dataset) MaybeHead() (types.Value, bool)

MaybeHead returns the current Head Commit of this Dataset, which contains the current root of the Dataset's value tree, if available. If not, it returns a new Commit and 'false'.

func (Dataset) MaybeHeadAddr 

func (ds Dataset) MaybeHeadAddr() (hash.Hash, bool)

func (Dataset) MaybeHeadRef 

func (ds Dataset) MaybeHeadRef() (types.Ref, bool, error)

MaybeHeadRef returns the Ref of the current Head Commit of this Dataset, which contains the current root of the Dataset's value tree, if available. If not, it returns an empty Ref and 'false'.

func (Dataset) MaybeHeadValue 

func (ds Dataset) MaybeHeadValue() (types.Value, bool, error)

MaybeHeadValue returns the Value field of the current head Commit, if available. If not it returns nil and 'false'.

func (Dataset) MaybeHeight 

func (ds Dataset) MaybeHeight() (uint64, bool, error)

type DatasetsMap 

type DatasetsMap interface {
	// How many datasets are in the map
	Len() uint64

	IterAll(ctx context.Context, cb func(id string, addr hash.Hash) error) error
}

type GarbageCollector 

type GarbageCollector interface {
	types.ValueReadWriter

	// GC traverses the database starting at the Root and removes
	// all unreferenced data from persistent storage.
	GC(ctx context.Context, oldGenRefs, newGenRefs hash.HashSet) error
}

GarbageCollector provides a method to remove unreferenced data from a store.

type MergeState 

type MergeState struct {
	// contains filtered or unexported fields
}

func NewMergeState 

func NewMergeState(ctx context.Context, vrw types.ValueReadWriter, preMergeWorking types.Ref, commit *Commit) (*MergeState, error)

func (*MergeState) FromCommit 

func (ms *MergeState) FromCommit(ctx context.Context, vr types.ValueReader) (*Commit, error)

func (*MergeState) PreMergeWorkingAddr 

func (ms *MergeState) PreMergeWorkingAddr(ctx context.Context, vr types.ValueReader) (hash.Hash, error)

type RefMapEdit 

type RefMapEdit struct {
	Name string
	Addr hash.Hash
}

type TagMeta 

type TagMeta struct {
	Name          string
	Email         string
	Timestamp     uint64
	Description   string
	UserTimestamp int64
}

TagMeta contains all the metadata that is associated with a tag within a data repo.

func NewTagMeta 

func NewTagMeta(name, email, desc string) *TagMeta

NewTagMetaWithUserTS returns TagMeta that can be used to create a tag. It uses the current time as the user timestamp.

func NewTagMetaWithUserTS 

func NewTagMetaWithUserTS(name, email, desc string, userTS time.Time) *TagMeta

NewTagMetaWithUserTS returns TagMeta that can be used to create a tag.

func (*TagMeta) FormatTS 

func (tm *TagMeta) FormatTS() string

FormatTS takes the internal timestamp and turns it into a human readable string in the time.RubyDate format which looks like: "Mon Jan 02 15:04:05 -0700 2006"

func (*TagMeta) String 

func (tm *TagMeta) String() string

String returns the human readable string representation of the tag data

func (*TagMeta) Time 

func (tm *TagMeta) Time() time.Time

Time returns the time at which the tag occurred

type TagOptions 

type TagOptions struct {
	// Meta is a Struct that describes arbitrary metadata about this Tag,
	// e.g. a timestamp or descriptive text.
	Meta *TagMeta
}

TagOptions is used to pass options into Tag.

type WorkingSetHead 

type WorkingSetHead struct {
	Meta        *WorkingSetMeta
	WorkingAddr hash.Hash
	StagedAddr  *hash.Hash
	MergeState  *MergeState
}

type WorkingSetMeta 

type WorkingSetMeta struct {
	Name        string
	Email       string
	Description string
	Timestamp   uint64
}

type WorkingSetSpec 

type WorkingSetSpec struct {
	Meta        *WorkingSetMeta
	WorkingRoot types.Ref
	StagedRoot  types.Ref
	MergeState  *MergeState
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.