sync

Documentation

Overview

Package sync implements a sync mechanism using Merkle trees (tree of hahes) for a two-way sync between two BlobStash instances.

The algorithm is inspired by Dynamo or Cassandra uses of Merkle trees (as an anti-entropy mechanism).

Each node maintains its own Merkle tree, when doing a sync, the hashes of the tree are checked against each other starting from the root hash to the leaves.

This first implementation only keep 256 (16**2) buckets (the first 2 hex of the hashes).

Blake2B (the same hashing algorithm used by the Blob Store) is used to compute the tree.

Index

• func NewHash() (h hash.Hash)
• type LeafState
• type State
  • func (st *State) String() string
• type StateTree
  • func NewStateTree() *StateTree
  • func (st *StateTree) Add(h string)
  • func (st *StateTree) Close() error
  • func (st *StateTree) Count() int
  • func (st *StateTree) Level1() map[string]string
  • func (st *StateTree) Level1Prefix(prefix string) string
  • func (st *StateTree) Root() string
  • func (st *StateTree) State() *State
  • func (st *StateTree) String() string
• type Sync
  • func New(logger log2.Logger, conf *config.Config, blobstore store.BlobStore) *Sync
  • func (st *Sync) Client(url, apiKey string, oneWay bool) *SyncClient
  • func (st *Sync) LeafState(prefix string) (*LeafState, error)
  • func (st *Sync) Register(r *mux.Router, basicAuth func(http.Handler) http.Handler)
  • func (st *Sync) Sync(url, apiKey string, oneWay bool) (*SyncStats, error)
• type SyncClient
  • func NewSyncClient(logger log.Logger, st *Sync, state *StateTree, blobstore store.BlobStore, ...) *SyncClient
  • func (stc *SyncClient) Receive(h string) error
  • func (stc *SyncClient) RemoteLeaf(prefix string) (*LeafState, error)
  • func (stc *SyncClient) RemoteState() (*State, error)
  • func (stc *SyncClient) Send(h string) error
  • func (stc *SyncClient) Sync() (*SyncStats, error)
• type SyncStats

Functions

func NewHash

func NewHash() (h hash.Hash)

Types

type LeafState

type LeafState struct {
	Prefix string   `json:"prefix"`
	Count  int      `json:"count"`
	Hashes []string `json:"hashes"`
}

type State

type State struct {
	Root   string            `json:"root"`
	Count  int               `json:"count"`
	Leaves map[string]string `json:"leaves"`
}

func (*State) String

func (st *State) String() string

type StateTree

type StateTree struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewStateTree

func NewStateTree() *StateTree

func (*StateTree) Add

func (st *StateTree) Add(h string)

func (*StateTree) Close

func (st *StateTree) Close() error

func (*StateTree) Count

func (st *StateTree) Count() int

func (*StateTree) Level1

func (st *StateTree) Level1() map[string]string

func (*StateTree) Level1Prefix

func (st *StateTree) Level1Prefix(prefix string) string

func (*StateTree) Root

func (st *StateTree) Root() string

func (*StateTree) State

func (st *StateTree) State() *State

func (*StateTree) String

func (st *StateTree) String() string

type Sync

type Sync struct {
	// contains filtered or unexported fields
}

func New

func New(logger log2.Logger, conf *config.Config, blobstore store.BlobStore) *Sync

func (*Sync) Client

func (st *Sync) Client(url, apiKey string, oneWay bool) *SyncClient

func (*Sync) LeafState

func (st *Sync) LeafState(prefix string) (*LeafState, error)

func (*Sync) Register

func (st *Sync) Register(r *mux.Router, basicAuth func(http.Handler) http.Handler)

func (*Sync) Sync

func (st *Sync) Sync(url, apiKey string, oneWay bool) (*SyncStats, error)

type SyncClient

type SyncClient struct {
	// contains filtered or unexported fields
}

func NewSyncClient

func NewSyncClient(logger log.Logger, st *Sync, state *StateTree, blobstore store.BlobStore, url, apiKey string, oneWay bool) *SyncClient

func (*SyncClient) Receive

func (stc *SyncClient) Receive(h string) error

func (*SyncClient) RemoteLeaf

func (stc *SyncClient) RemoteLeaf(prefix string) (*LeafState, error)

func (*SyncClient) RemoteState

func (stc *SyncClient) RemoteState() (*State, error)

func (*SyncClient) Send

func (stc *SyncClient) Send(h string) error

func (*SyncClient) Sync

func (stc *SyncClient) Sync() (*SyncStats, error)

type SyncStats

type SyncStats struct {
	Downloaded     int    `json:"blobs_downloaded"`
	DownloadedSize int    `json:"downloaded_size"`
	Uploaded       int    `json:"blobs_uploaded"`
	UploadedSize   int    `json:"uploaded_size"`
	Duration       string `json:"sync_duration"`
	AlreadySynced  bool   `json:"already_in_sync"`
	OneWay         bool   `json:"one_way_sync"`
}