merger

README

dfuse Merger

The merger process is responsible for accumulating blocks from all forks visible by the pool of instrumented nodes, and builds the famoux 100-blocks files consumed by bstream's FileSource and may other dfuse processes.

Installation & Usage

See the different protocol-specific dfuse binaries at https://github.com/dfuse-io/dfuse#protocols

Current implementations:

• dfuse for EOSIO
• dfuse for Ethereum, soon to be open sourced

Contributing

Issues and PR in this repo related strictly to the merger functionalities

Report any protocol-specific issues in their respective repositories

Please first refer to the general dfuse contribution guide, if you wish to contribute to this code base.

License

Apache 2.0

Documentation

Index

• Variables
• func Retry(attempts int, sleep time.Duration, callback func() error) (err error)
• type Bundle
  • func NewBundle(lowerBlockNum, chunkSize uint64) *Bundle
• type Merger
  • func NewMerger(sourceStore dstore.Store, destStore dstore.Store, ...) *Merger
  • func (m *Merger) CacheInvalid() bool
  • func (m *Merger) Check(ctx context.Context, in *pbhealth.HealthCheckRequest) (*pbhealth.HealthCheckResponse, error)
  • func (m *Merger) FindNextBaseBlock() (uint64, error)
  • func (m *Merger) Launch()
  • func (m *Merger) PreMergedBlocks(ctx context.Context, req *pbmerge.Request) (*pbmerge.Response, error)
  • func (m *Merger) SetupBundle(start, stop uint64)
• type OneBlockFile
• type SeenBlockCache
  • func NewSeenBlockCache(filename string, keepSize uint64) (c *SeenBlockCache)
  • func (c *SeenBlockCache) Add(filename string)
  • func (c *SeenBlockCache) IsTooOld(num uint64) bool
  • func (c *SeenBlockCache) Reset()
  • func (c *SeenBlockCache) Save() error
  • func (c *SeenBlockCache) SeenBefore(filename string) bool
  • func (c *SeenBlockCache) Truncate()

Variables

var DeleteObjectTimeout = 5 * time.Minute

var GetObjectTimeout = 5 * time.Minute

var ListFilesTimeout = 10 * time.Minute

var WriteObjectTimeout = 5 * time.Minute

Functions

func Retry

func Retry(attempts int, sleep time.Duration, callback func() error) (err error)

Types

type Bundle

type Bundle struct {
	// contains filtered or unexported fields
}

func NewBundle

func NewBundle(lowerBlockNum, chunkSize uint64) *Bundle

type Merger

type Merger struct {
	*shutter.Shutter
	// contains filtered or unexported fields
}

func NewMerger

func NewMerger(
	sourceStore dstore.Store,
	destStore dstore.Store,
	writersLeewayDuration time.Duration,
	minimalBlockNum uint64,
	progressFilename string,
	deleteBlocksBefore bool,
	seenCacheFilename string,
	timeBetweenStoreLookups time.Duration,
	maxFixableFork uint64,
	grpcListenAddr string) *Merger

func (*Merger) CacheInvalid

func (m *Merger) CacheInvalid() bool

func (*Merger) Check

func (m *Merger) Check(ctx context.Context, in *pbhealth.HealthCheckRequest) (*pbhealth.HealthCheckResponse, error)

func (*Merger) FindNextBaseBlock

func (m *Merger) FindNextBaseBlock() (uint64, error)

findNextBaseBlock will return an error if there is a gap found ...

func (*Merger) Launch

func (m *Merger) Launch()

func (*Merger) PreMergedBlocks

func (m *Merger) PreMergedBlocks(ctx context.Context, req *pbmerge.Request) (*pbmerge.Response, error)

func (*Merger) SetupBundle

func (m *Merger) SetupBundle(start, stop uint64)

type OneBlockFile

type OneBlockFile struct {
	// contains filtered or unexported fields
}

type SeenBlockCache

type SeenBlockCache struct {
	M map[string]bool

	HighestSeen uint64
	// contains filtered or unexported fields
}

func NewSeenBlockCache

func NewSeenBlockCache(filename string, keepSize uint64) (c *SeenBlockCache)

func (*SeenBlockCache) Add

func (c *SeenBlockCache) Add(filename string)

func (*SeenBlockCache) IsTooOld

func (c *SeenBlockCache) IsTooOld(num uint64) bool

func (*SeenBlockCache) Reset

func (c *SeenBlockCache) Reset()

func (*SeenBlockCache) Save

func (c *SeenBlockCache) Save() error

func (*SeenBlockCache) SeenBefore

func (c *SeenBlockCache) SeenBefore(filename string) bool

func (*SeenBlockCache) Truncate

func (c *SeenBlockCache) Truncate()