crawler

Documentation

Index

• Variables
• func CheckVariablesForCrawler() error
• type BlockchainState
  • func (bs *BlockchainState) GetLatestBlockNumber() *big.Int
  • func (bs *BlockchainState) GetLatestUpdateTs() time.Time
  • func (bs *BlockchainState) RaiseLatestBlockNumber(blockNumber *big.Int)
• type CrawlPack
  • func (cp *CrawlPack) Initialize(startBlock int64)
  • func (cp *CrawlPack) ProcessAndPush(client seer_blockchain.BlockchainClient, crawler *Crawler) error
• type Crawler
  • func NewCrawler(blockchain, rpcUrl string, ...) (*Crawler, error)
  • func (c *Crawler) Start(threads int)
• type DynamicBatch
  • func (db *DynamicBatch) DynamicDecreaseSize(diff int64)
  • func (db *DynamicBatch) GetSize() int64

Variables

var (
	SeerDefaultBlockShift    int64  = 100
	SeerCrawlerStoragePrefix string = "dev"

	SEER_CRAWLER_DEBUG = false
)

Functions

func CheckVariablesForCrawler

func CheckVariablesForCrawler() error

CheckVariablesForCrawler checks required environment variables but only for the specified chain.

Types

type BlockchainState

type BlockchainState struct {
	LatestUpdateTs    time.Time
	LatestBlockNumber *big.Int
	// contains filtered or unexported fields
}

BlockchainState represents the current state of the blockchain, including the latest block number and the time when it was fetched.

var CurrentBlockchainState BlockchainState

func (*BlockchainState) GetLatestBlockNumber

func (bs *BlockchainState) GetLatestBlockNumber() *big.Int

func (*BlockchainState) GetLatestUpdateTs

func (bs *BlockchainState) GetLatestUpdateTs() time.Time

func (*BlockchainState) RaiseLatestBlockNumber

func (bs *BlockchainState) RaiseLatestBlockNumber(blockNumber *big.Int)

type CrawlPack

type CrawlPack struct {
	PackSize         int64
	PackCrawlStartTs time.Time

	BlocksPack      []proto.Message
	BlocksIndexPack []indexer.BlockIndex

	PackStartBlock int64
	PackEndBlock   int64
	// contains filtered or unexported fields
}

func (*CrawlPack) Initialize

func (cp *CrawlPack) Initialize(startBlock int64)

func (*CrawlPack) ProcessAndPush

func (cp *CrawlPack) ProcessAndPush(client seer_blockchain.BlockchainClient, crawler *Crawler) error

ProcessAndPush makes preparations for blocks, txs, and logs data and pushes them to the database with storage.

type Crawler

type Crawler struct {
	Client          seer_blockchain.BlockchainClient
	StorageInstance storage.Storer
	// contains filtered or unexported fields
}

Crawler defines the crawler structure.

func NewCrawler

func NewCrawler(blockchain, rpcUrl string, startBlock, finalBlock, confirmations, batchSize int64, timeout int, baseDir string, protoSizeLimit uint64, protoTimeLimit, retryWait, retryMultiplier int) (*Crawler, error)

NewCrawler creates a new crawler instance with the given blockchain handler.

func (*Crawler) Start

func (c *Crawler) Start(threads int)

Main crawler loop.

type DynamicBatch

type DynamicBatch struct {
	Size int64
	// contains filtered or unexported fields
}

DynamicBatch is used to dynamically calculate the batch size of blocks in order to fetch them.

func (*DynamicBatch) DynamicDecreaseSize

func (db *DynamicBatch) DynamicDecreaseSize(diff int64)

DynamicDecreaseSize by dividing it by 2 and checking if it is not below the diff argument.

func (*DynamicBatch) GetSize

func (db *DynamicBatch) GetSize() int64