merger

package
v2.2.0-rc2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 17, 2020 License: AGPL-3.0 Imports: 23 Imported by: 46

Documentation

Overview

Package merger implements all logic to compare trees and create set of operations to be applied

Index

Constants

View Source
const (
	NodeType_METADATA tree.NodeType = 3

	MetaNodeParentUUIDMeta = "ParentUUID"
	MetaNodeParentPathMeta = "ParentPath"
)

Variables

This section is empty.

Functions

func MostRecentNode

func MostRecentNode(n1, n2 *tree.Node) *tree.Node

MostRecentNode compares two nodes Modification Time and returns the most recent one

Types

type AbstractPatch

type AbstractPatch struct {
	// contains filtered or unexported fields
}

func (*AbstractPatch) Done

func (b *AbstractPatch) Done(info interface{})

func (*AbstractPatch) FinishSession

func (b *AbstractPatch) FinishSession(sessionUuid string) (err error)

func (*AbstractPatch) FlushSession

func (b *AbstractPatch) FlushSession(sessionUuid string) error

func (*AbstractPatch) GetStamp

func (b *AbstractPatch) GetStamp() time.Time

func (*AbstractPatch) GetUUID

func (b *AbstractPatch) GetUUID() string

func (*AbstractPatch) HasTransfers

func (b *AbstractPatch) HasTransfers() bool

func (*AbstractPatch) PostFilter

func (b *AbstractPatch) PostFilter(f ...func() error) []func() error

PostFilter gets or sets a callback to be triggered after filtering

func (*AbstractPatch) SetError

func (b *AbstractPatch) SetError(e error)

Set a global error status on this patch

func (*AbstractPatch) SetLockSessionData

func (b *AbstractPatch) SetLockSessionData(providerContext context.Context, lockSession string)

func (*AbstractPatch) SetPatchError

func (b *AbstractPatch) SetPatchError(e error) error

func (*AbstractPatch) SetSessionData

func (b *AbstractPatch) SetSessionData(providerContext context.Context, silentSession bool)

func (*AbstractPatch) SetUUID

func (b *AbstractPatch) SetUUID(u string)

func (*AbstractPatch) SetupChannels

func (b *AbstractPatch) SetupChannels(status chan model.Status, done chan interface{}, cmd *model.Command)

func (*AbstractPatch) SkipFilterToTarget

func (b *AbstractPatch) SkipFilterToTarget(skip bool)

func (*AbstractPatch) Source

func (b *AbstractPatch) Source(newSource ...model.PathSyncSource) model.PathSyncSource

func (*AbstractPatch) Stamp

func (b *AbstractPatch) Stamp(t time.Time)

func (*AbstractPatch) StartSession

func (b *AbstractPatch) StartSession(rootNode *tree.Node) (*tree.IndexationSession, error)

func (*AbstractPatch) Status

func (b *AbstractPatch) Status(s model.Status)

func (*AbstractPatch) Target

func (b *AbstractPatch) Target(newTarget ...model.PathSyncTarget) model.PathSyncTarget

type BidirectionalPatch

type BidirectionalPatch struct {
	TreePatch
	// contains filtered or unexported fields
}

BidirectionalPatch is a Patch that can handle operations going both left and right side. It handles conflicts detection and tries to solve them automatically if possible

func ComputeBidirectionalPatch

func ComputeBidirectionalPatch(ctx context.Context, left, right Patch) (*BidirectionalPatch, error)

ComputeBidirectionalPatch merges two unidirectional Patch into one BidirectionalPatch

func NewBidirectionalPatch

func NewBidirectionalPatch(ctx context.Context, source, target model.Endpoint) *BidirectionalPatch

func (*BidirectionalPatch) AppendBranch

func (p *BidirectionalPatch) AppendBranch(ctx context.Context, branch *BidirectionalPatch)

AppendBranch merges another bidir patch into this existing patch

func (*BidirectionalPatch) Filter

func (p *BidirectionalPatch) Filter(ctx context.Context, ignores ...glob.Glob)

Filter override TreePatch.Filter and does nothing. Left and right patches are filtered at BidirectionalPatch creation time.

func (*BidirectionalPatch) FinishSession

func (p *BidirectionalPatch) FinishSession(sessionUuid string) error

FinishSession overrides AbstractPatch method to handle source and/or target as session provider

func (*BidirectionalPatch) FlushSession

func (p *BidirectionalPatch) FlushSession(sessionUuid string) error

FlushSession overrides AbstractPatch method to handle source and/or target as session provider

func (*BidirectionalPatch) StartSession

func (p *BidirectionalPatch) StartSession(rootNode *tree.Node) (*tree.IndexationSession, error)

StartSession overrides AbstractPatch method to handle source and/or target as session provider

type ChildrenCursor

type ChildrenCursor struct {
	// contains filtered or unexported fields
}

ChildrenCursor provides a Nexter for browsing a node children

func (*ChildrenCursor) Next

func (c *ChildrenCursor) Next() *TreeNode

Next sends the next child or nil

type ConflictOperation

type ConflictOperation interface {
	ConflictInfo() (t ConflictType, left Operation, right Operation)
}

type ConflictType

type ConflictType int
const (
	ConflictFolderUUID ConflictType = iota
	ConflictFileContent
	ConflictNodeType
	ConflictPathOperation
	ConflictMoveSameSource
	ConflictMoveSameTarget
	ConflictMetaChanged
)

type Diff

type Diff interface {
	model.Stater
	model.StatusProvider

	// Compute performs the actual Diff operation
	Compute(root string, lock chan bool, rootStats map[string]*model.EndpointRootStat, ignores ...glob.Glob) error
	// ToUnidirectionalPatch transforms current diff into a set of patch operations
	ToUnidirectionalPatch(direction model.DirectionType, patch Patch) (err error)
	// ToBidirectionalPatch transforms current diff into a bidirectional patch of operations
	ToBidirectionalPatch(leftTarget model.PathSyncTarget, rightTarget model.PathSyncTarget, patch *BidirectionalPatch) (err error)
}

Diff represents basic differences between two sources It can be then transformed to Patch, depending on the sync being unidirectional (transform to Creates and Deletes) or bidirectional (transform only to Creates)

func NewDiff

func NewDiff(ctx context.Context, left model.PathSyncSource, right model.PathSyncSource) Diff

NewDiff creates a new Diff implementation

type DiffConflict

type DiffConflict struct {
	Type      ConflictType
	NodeLeft  *tree.Node
	NodeRight *tree.Node
}

Conflict represent a conflict between two nodes at the same path

type MetaConfig

type MetaConfig struct {
	MetaNames  []string
	MetaRegexp []*regexp.Regexp
}

type Move

type Move struct {
	// contains filtered or unexported fields
}

func (*Move) MarshalLogObject

func (m *Move) MarshalLogObject(encoder zapcore.ObjectEncoder) error

func (*Move) SameBase

func (m *Move) SameBase() bool

type OpWalker

type OpWalker func(Operation)

OpWalker is a callback passed to the Walk functions of a patch

type Operation

type Operation interface {
	fmt.Stringer

	// Type describes the operation type
	Type() OperationType
	// GetNode returns the attached node. For a create, it is the original node, for a delete or a move,
	// it is the node that is to be deleted/moved inside the target.
	GetNode() *tree.Node
	// IsScanEvent tells whether this operation was triggered by a manual scan (true) or by a Watch event (false)
	IsScanEvent() bool
	// IsTypeMove is a shortcut for OpMoveFile / OpMoveFolder types
	IsTypeMove() bool
	// IsTypeData is a shortcut for OpCreateFile / OpUpdateFile types
	IsTypeData() bool
	// IsTypePath is a shortcut for non-data operations
	IsTypePath() bool
	// IsProcessed tells wether this operation has been succesfully processed or not.
	IsProcessed() bool
	// Error returns any error attached to this operation processing
	Error() error
	// ErrorString returns the error as a string
	ErrorString() string
	// CleanError removes current error - it is called before reprocessing an operation
	CleanError()
	// Status sets the last known status of this operation
	Status(status model.Status)
	// GetStatus returns the last known status of this operation
	GetStatus() model.Status
	// GetRefPath returns the reference path used to check a node being present or not. For a move, it is the target path.
	// This path is dynamically computed based on the the parent operations being already processed or not.
	GetRefPath() string
	// GetMoveOriginPath returns the source path if operation is a move.
	// This path is dynamically computed based on the the parent operations being already processed or not.
	GetMoveOriginPath() string

	// SetProcessed flags operation as succesfully processed
	SetProcessed()
	// SetDirection updates the operation direction. This is use for BiDirectionalPatches that can contain operations to be
	// applied in both directions
	SetDirection(OperationDirection) Operation
	// SetNode updates the reference node
	SetNode(n *tree.Node)
	// UpdateRefPath updates the reference path
	UpdateRefPath(p string)
	// UpdateMoveOriginPath updates the origin path for a move
	UpdateMoveOriginPath(p string)
	// UpdateType changes the internal type
	UpdateType(t OperationType)

	// AttachToPath links this operation to a given patch
	AttachToPatch(p Patch)
	// Source returns the operation source from the attached Patch
	Source() model.PathSyncSource
	// Target returns the operation target from the attached Patch
	Target() model.PathSyncTarget
	// NodeFromSource tries to load the node from the attached Patch source
	NodeFromSource(ctx context.Context) (node *tree.Node, err error)
	// NodeInTarget tries to find the node in the attached Patch target
	NodeInTarget(ctx context.Context, cache ...model.PathSyncSource) (node *tree.Node, found bool)

	// Clone creates a clone of this operation, eventually replacing its type.
	Clone(replaceType ...OperationType) Operation
	// CreateContext creates an appropriate context to be used by a Processor to pass useful informations to endpoints
	CreateContext(ctx context.Context) context.Context
}

Operation describes an atomic operation to be passed to a processor and applied to an endpoint

func NewConflictOperation

func NewConflictOperation(node *tree.Node, t ConflictType, left, right Operation) Operation

func NewOpForUnmarshall

func NewOpForUnmarshall() Operation

func NewOperation

func NewOperation(t OperationType, e model.EventInfo, loadedNode ...*tree.Node) Operation

type OperationDirection

type OperationDirection int
const (
	OperationDirDefault OperationDirection = iota
	OperationDirLeft
	OperationDirRight
)

type OperationType

type OperationType int

OperationType describes the type of operation to be applied

const (
	OpCreateFile OperationType = iota
	OpUpdateFile
	OpCreateFolder
	OpMoveFolder
	OpMoveFile
	OpDelete
	OpRefreshUuid
	OpConflict
	OpUnknown
	OpDeleteMeta
	OpCreateMeta
	OpUpdateMeta
)
const OpMoveTarget OperationType = 101
const OpNone OperationType = 100

func (OperationType) String

func (t OperationType) String() string

String gives a string representation of this integer type

type Patch

type Patch interface {
	model.Stater
	model.StatusProvider

	// GetUUID provides a unique ID for this patch
	GetUUID() string
	// SetUUID set the uuid
	SetUUID(string)
	// GetStamp returns a last modified date
	GetStamp() time.Time
	// Stamp set the last modified date on this patch
	Stamp(time.Time)

	// Source get or set the source of this patch
	Source(newSource ...model.PathSyncSource) model.PathSyncSource
	// Target get or set the target of this patch
	Target(newTarget ...model.PathSyncTarget) model.PathSyncTarget

	// Enqueue stacks a Operation - By default, it is registered with the event.Key, but an optional key can be passed.
	Enqueue(event Operation)
	// WalkOperations crawls operations in correct order, with an optional filter (no filter = all operations)
	WalkOperations(opTypes []OperationType, callback OpWalker)
	// EventsByTypes retrieves all events of a given type
	OperationsByType(types []OperationType, sorted ...bool) (events []Operation)

	// Filter tries to detect unnecessary changes locally
	Filter(ctx context.Context, ignores ...glob.Glob)
	// FilterToTarget tries to compare changes to target and remove unnecessary ones
	FilterToTarget(ctx context.Context)
	// SkipTargetChecks set a flag to skip FilterToTarget
	SkipFilterToTarget(bool)
	// PostFilter gets or sets a callback to be triggered after filtering
	PostFilter(...func() error) []func() error
	// Validate browses target to verify all changes are correctly reflected (and indexed)
	Validate(ctx context.Context) error

	// HasTransfers tels if the source and target will exchange actual data.
	HasTransfers() bool
	// Size returns the total number of operations
	Size() int
	// ProgressTotal returns the total number of bytes to be processed, to be used for progress.
	// Basically, file transfers operations returns the file size, but other operations return a 1 byte size.
	ProgressTotal() int64

	// SetPatchError sets a global error on this patch
	SetPatchError(e error) error
	// HasErrors checks if this patch has a global error status
	HasErrors() ([]error, bool)
	// CleanErrors cleans errors from patch before trying to reapply it
	CleanErrors()

	// SetSessionData sets some internal information to be used if Source or Target
	// implement the SessionProvider interface
	SetSessionData(providerContext context.Context, silentSession bool)
	// SetLockSessionData sets some internal information to be used if Source or Target
	// implement the LockBranchProvider interface
	SetLockSessionData(providerContext context.Context, lockSession string)
	// StartSession calls StartSession on the underlying provider if it is set
	StartSession(rootNode *tree.Node) (*tree.IndexationSession, error)
	// FlushSession calls FlushSession on the underlying provider if it is set
	FlushSession(sessionUuid string) error
	// FinishSession calls FinishSession on the underlying provider if it is set
	FinishSession(sessionUuid string) error
}

Patch represents a set of operations to be processed

func ClonePatch

func ClonePatch(source model.PathSyncSource, target model.PathSyncTarget, origin Patch) Patch

ClonePatch creates a new patch with the same operations but different source/targets

func NewPatch

func NewPatch(source model.PathSyncSource, target model.PathSyncTarget, options PatchOptions) Patch

NewPatch creates a new Patch implementation

type PatchListener

type PatchListener interface {
	PublishPatch(patch Patch)
}

PatchListener has a PublishPatch method

type PatchOptions

type PatchOptions struct {
	MoveDetection bool
	NoRescan      bool
}

PatchOptions contains various options for initializing a patch

type Solver

type Solver func(left, right *TreeNode)

type TreeDiff

type TreeDiff struct {
	// contains filtered or unexported fields
}

Diff represent basic differences between two sources It can be then transformed to Patch, depending on the sync being unidirectional (transform to Creates and Deletes) or bidirectional (transform only to Creates)

func NewTreeDiff

func NewTreeDiff(ctx context.Context, left model.PathSyncSource, right model.PathSyncSource) *TreeDiff

NewTreeDiff is a public access for instantiating a new TreeDiff

func (*TreeDiff) Compute

func (diff *TreeDiff) Compute(root string, lock chan bool, rootStats map[string]*model.EndpointRootStat, ignores ...glob.Glob) error

Compute performs the actual diff between left and right

func (*TreeDiff) Done

func (diff *TreeDiff) Done(info interface{})

func (*TreeDiff) SetupChannels

func (diff *TreeDiff) SetupChannels(status chan model.Status, done chan interface{}, cmd *model.Command)

SetupChannels registers status chan internally. Done chan is ignored

func (*TreeDiff) Stats

func (diff *TreeDiff) Stats() map[string]interface{}

Stats provides info about the diff internals

func (*TreeDiff) Status

func (diff *TreeDiff) Status(status model.Status)

Status sends status to internal channel

func (*TreeDiff) String

func (diff *TreeDiff) String() string

String provides a string representation of this diff

func (*TreeDiff) ToBidirectionalPatch

func (diff *TreeDiff) ToBidirectionalPatch(leftTarget model.PathSyncTarget, rightTarget model.PathSyncTarget, patch *BidirectionalPatch) (err error)

ToBidirectionalPatch computes a bidirectional patch from this diff using the given targets

func (*TreeDiff) ToUnidirectionalPatch

func (diff *TreeDiff) ToUnidirectionalPatch(direction model.DirectionType, patch Patch) (err error)

ToUnidirectionalPatch transforms this diff to a patch

type TreeNode

type TreeNode struct {
	tree.Node
	sync.Mutex

	// PathOperation defines an operation on path, like mkdir, move, delete...
	PathOperation Operation
	// DataOperation defines an operation on data transfer
	DataOperation Operation
	// Conflict encapsulates two conflicting operations on the same node
	Conflict Operation
	// OpMoveTarget is a reference to the target node for move operations
	OpMoveTarget *TreeNode
	// MoveSourcePath is a reference to the source node path if there is a move operation (necessary for detecting conflicts)
	MoveSourcePath string
	// contains filtered or unexported fields
}

TreeNode builds a Merkle Tree but with N children and the ability to compute the COLLECTION Nodes hashes to detect changes in branches more rapidly

func NewTree

func NewTree() *TreeNode

func NewTreeNode

func NewTreeNode(n *tree.Node) *TreeNode

NewTreeNode creates a new node from a tree.Node. Can be a root, a COLL or a LEAF.

func TreeNodeFromSource

func TreeNodeFromSource(source model.PathSyncSource, root string, ignores []glob.Glob, includeMetas []glob.Glob, status ...chan model.Status) (*TreeNode, error)

TreeNodeFromSource populates a hash tree with leafs and folders by walking a source. When it comes accross a LEAF without Etag value, it asks the source to recompute it in a parallel fashion with throttling (max 15 at the same time). At the end of the operation, the tree should be fully loaded with all LEAF etags (but not COLL etags).

func (*TreeNode) AddChild

func (t *TreeNode) AddChild(n *TreeNode)

AddChild appends a child to the children map (with lock)

func (*TreeNode) ChildByPath

func (t *TreeNode) ChildByPath(p string) *TreeNode

func (*TreeNode) ClearChildren

func (t *TreeNode) ClearChildren()

func (*TreeNode) Enqueue

func (t *TreeNode) Enqueue(nodes []*tree.Node) []*tree.Node

Enqueue recursively appends al tree.Node and the children's tree.Node to a slice

func (*TreeNode) GetCursor

func (t *TreeNode) GetCursor() *ChildrenCursor

GetCursor gives a cursor to crawl the current node children

func (*TreeNode) GetHash

func (t *TreeNode) GetHash() string

GetHash returns the Etag of the node. For leaf it should be available, for Folders if it is not already computed, it will compute an etag from the children recursively, using their name and Etag.

func (*TreeNode) GetLevel

func (t *TreeNode) GetLevel() int

GetLevel computes the current level of this node (depth)

func (*TreeNode) Label

func (t *TreeNode) Label() string

Label returns the basename of the path

func (*TreeNode) MarshalJSON

func (t *TreeNode) MarshalJSON() ([]byte, error)

MarshalJSON serializes specific fields for output to JSON

func (*TreeNode) OriginalPath

func (t *TreeNode) OriginalPath() string

OriginalPath rebuilds node Path climbing to the root

func (*TreeNode) ParentPath

func (t *TreeNode) ParentPath() string

ParentPath returns the parent Dir path

func (*TreeNode) PrintTree

func (t *TreeNode) PrintTree() string

PrintOut sends to fmt.Println a tree version of this node

func (*TreeNode) ProcessedPath

func (t *TreeNode) ProcessedPath(asProcessed bool, isNext ...bool) string

ProcessedPath builds node Path to the root taking all moves into account

func (*TreeNode) PruneIdentityPathOperation

func (t *TreeNode) PruneIdentityPathOperation() bool

PruneIdentityPathOperation detects if this PathOperation will result in Identity, remove it in that case.

func (*TreeNode) QueueOperation

func (t *TreeNode) QueueOperation(op Operation)

QueueOperation registers an operation at a given path, by eventually building traversing nodes without operations on them

func (*TreeNode) SortedChildren

func (t *TreeNode) SortedChildren() []*TreeNode

SortedChildren sorts children by their labels. An internal flag avoids resorting if it was already sorted once.

func (*TreeNode) Walk

func (t *TreeNode) Walk(cb func(n *TreeNode) bool)

func (*TreeNode) WalkOperations

func (t *TreeNode) WalkOperations(opTypes []OperationType, callback OpWalker)

WalkOperations walks the tree looking for operation of a certain type

func (*TreeNode) WalkToFirstOperations

func (t *TreeNode) WalkToFirstOperations(opType OperationType, callback func(Operation), target ...model.Endpoint)

WalkToFirstOperations walks the tree (depth-first) and stops on a branch as soon as it finds a given operation Type

type TreePatch

type TreePatch struct {
	AbstractPatch
	TreeNode
	// contains filtered or unexported fields
}

TreePatch is an implement of the Patch interface representing a sequence of operations as a tree structure. It is based on a TreeNode: each node can eventually contain a PathOperation or a DataOperation, or no operation at all if they are just for traversing.

MOVES operation will add two nodes (and their traversing parents if required) to the tree, for both the Origin and the Target.

Operations paths are computed dynamically based on the state of the parents (whether they have been processed or not). That way, if a move is applied at any level, the operations of the children always return the correct origin/target paths.

func (*TreePatch) BranchesWithOperations

func (t *TreePatch) BranchesWithOperations(endpoint model.Endpoint) (branches []string)

BranchesWithOperations finds highest modified paths. It walks the tree to find the first nodes having operations, and returns their parent folder.

func (*TreePatch) CachedBranchFromEndpoint

func (t *TreePatch) CachedBranchFromEndpoint(ctx context.Context, endpoint model.Endpoint) (model.PathSyncSource, bool)

CachedBranchFromEndpoint will walk to the first operations to find the branches containing some modifications

func (*TreePatch) CleanErrors

func (t *TreePatch) CleanErrors()

func (*TreePatch) Enqueue

func (t *TreePatch) Enqueue(op Operation)

Enqueue adds an operation to this patch

func (*TreePatch) Filter

func (t *TreePatch) Filter(ctx context.Context, ignores ...glob.Glob)

func (*TreePatch) FilterToTarget

func (t *TreePatch) FilterToTarget(ctx context.Context)

FilterToTarget tries to detect unnecessary operations based on the target status. If the target implements the CachedBranchProvider interface, instead of stat'ing the nodes one by one, the target will be fully loaded in memory at once to be used as a comparision.

func (*TreePatch) HasErrors

func (t *TreePatch) HasErrors() (errs []error, has bool)

HasErrors checks if this patch has a global error status or any operation in Error state

func (*TreePatch) HasTransfers

func (t *TreePatch) HasTransfers() bool

HasTransfers looks for create/update files between DataSyncTargets It keeps an internal state to avoid re-walking the tree unnecessarily each time it is called on a patch

func (*TreePatch) MarshalJSON

func (t *TreePatch) MarshalJSON() ([]byte, error)

MarshalJSON implements custom JSON marshalling

func (*TreePatch) OperationsByType

func (t *TreePatch) OperationsByType(types []OperationType, sorted ...bool) (events []Operation)

OperationsByType collects operations for a given type and return them in a slice

func (*TreePatch) ProgressTotal

func (t *TreePatch) ProgressTotal() int64

func (*TreePatch) Size

func (t *TreePatch) Size() int

Size returns the size of all operations

func (*TreePatch) Stats

func (t *TreePatch) Stats() map[string]interface{}

func (*TreePatch) String

func (t *TreePatch) String() string

func (*TreePatch) Validate

func (t *TreePatch) Validate(ctx context.Context) error

Validate tries to match the status of the target to verify that all operations are correctly applied.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL