snapshot

package
v1.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 29, 2022 License: Apache-2.0 Imports: 33 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// The supported snapshot file version.
	SupportedFormatVersion byte = 1
	// The length of a solid entry point hash.
	SolidEntryPointHashLength = iotago.MessageIDLength
)

Variables

View Source 
var (
	// Returned when a critical error stops the execution of a task.
	ErrCritical = errors.New("critical error")
	// Returned when unsupported snapshot data is read.
	ErrUnsupportedSnapshot = errors.New("unsupported snapshot data")
	// Returned when a child message wasn't found.
	ErrChildMsgNotFound = errors.New("child message not found")
	// Returned when the milestone diff that should be applied is not the current or next milestone.
	ErrWrongMilestoneDiffIndex = errors.New("wrong milestone diff index")
	// Returned when the final milestone after loading the snapshot is not equal to the solid entry point index.
	ErrFinalLedgerIndexDoesNotMatchSEPIndex = errors.New("final ledger index does not match solid entry point index")
	// Returned when a delta snapshot is available, but no full snapshot is found.
	ErrInvalidSnapshotAvailabilityState = errors.New("invalid snapshot files availability")

	ErrNoSnapshotSpecified                   = errors.New("no snapshot file was specified in the config")
	ErrNoSnapshotDownloadURL                 = errors.New("no download URL specified for snapshot files in config")
	ErrSnapshotDownloadWasAborted            = errors.New("snapshot download was aborted")
	ErrSnapshotDownloadNoValidSource         = errors.New("no valid source found, snapshot download not possible")
	ErrSnapshotCreationWasAborted            = errors.New("operation was aborted")
	ErrSnapshotCreationFailed                = errors.New("creating snapshot failed")
	ErrTargetIndexTooNew                     = errors.New("snapshot target is too new")
	ErrTargetIndexTooOld                     = errors.New("snapshot target is too old")
	ErrNotEnoughHistory                      = errors.New("not enough history")
	ErrNoPruningNeeded                       = errors.New("no pruning needed")
	ErrPruningAborted                        = errors.New("pruning was aborted")
	ErrDatabaseCompactionNotSupported        = errors.New("database compaction not supported")
	ErrDatabaseCompactionRunning             = errors.New("database compaction is running")
	ErrExistingDeltaSnapshotWrongLedgerIndex = errors.New("existing delta ledger snapshot has wrong ledger index")
)
View Source 
var (
	// Returned when an output producer has not been provided.
	ErrOutputProducerNotProvided = errors.New("output producer is not provided")
	// Returned when an output consumer has not been provided.
	ErrOutputConsumerNotProvided = errors.New("output consumer is not provided")
	// Returned when the treasury output for a full snapshot has not been provided.
	ErrTreasuryOutputNotProvided = errors.New("treasury output is not provided")
	// Returned when a treasury output consumer has not been provided.
	ErrTreasuryOutputConsumerNotProvided = errors.New("treasury output consumer is not provided")
	// Returned if specified snapshots are not mergeable.
	ErrSnapshotsNotMergeable = errors.New("snapshot files not mergeable")
)

Functions

func LoadSnapshotFilesToStorage added in v1.1.0 

func LoadSnapshotFilesToStorage(ctx context.Context, dbStorage *storage.Storage, fullPath string, deltaPath ...string) (*ReadFileHeader, *ReadFileHeader, error)

LoadSnapshotFilesToStorage loads the snapshot files from the given file paths into the storage.

func PruningMetricsCaller 

func PruningMetricsCaller(handler interface{}, params ...interface{})

PruningMetricsCaller is used to signal updated pruning metrics.

func SnapshotMetricsCaller 

func SnapshotMetricsCaller(handler interface{}, params ...interface{})

SnapshotMetricsCaller is used to signal updated snapshot metrics.

func StreamSnapshotDataFrom 

func StreamSnapshotDataFrom(reader io.Reader,
	headerConsumer HeaderConsumerFunc,
	sepConsumer SEPConsumerFunc,
	outputConsumer OutputConsumerFunc,
	unspentTreasuryOutputConsumer UnspentTreasuryOutputConsumerFunc,
	msDiffConsumer MilestoneDiffConsumerFunc) error

StreamSnapshotDataFrom consumes a snapshot from the given reader. OutputConsumerFunc must not be nil if the snapshot is not a delta snapshot.

Types

type DownloadTarget 

type DownloadTarget struct {
	// URL of the full snapshot file.
	Full string `json:"full"`
	// URL of the delta snapshot file.
	Delta string `json:"delta"`
}

DownloadTarget holds URLs to a full and delta snapshot.

type Events 

type Events struct {
	SnapshotMilestoneIndexChanged *events.Event
	SnapshotMetricsUpdated        *events.Event
	PruningMilestoneIndexChanged  *events.Event
	PruningMetricsUpdated         *events.Event
}

type FileHeader 

type FileHeader struct {
	// Version denotes the version of this snapshot.
	Version byte
	// Type denotes the type of this snapshot.
	Type Type
	// The ID of the network for which this snapshot is compatible with.
	NetworkID uint64
	// The milestone index of the SEPs for which this snapshot was taken.
	SEPMilestoneIndex milestone.Index
	// The milestone index of the ledger data within the snapshot.
	LedgerMilestoneIndex milestone.Index
	// The treasury output existing for the given ledger milestone index.
	// This field must be populated if a Full snapshot is created/read.
	TreasuryOutput *utxo.TreasuryOutput
}

FileHeader is the file header of a snapshot file.

type HeaderConsumerFunc 

type HeaderConsumerFunc func(*ReadFileHeader) error

HeaderConsumerFunc consumes the snapshot file header. A returned error signals to cancel further reading.

type LexicalOrderedOutputs added in v1.1.0 

type LexicalOrderedOutputs []*Output

LexicalOrderedOutputs are Outputs ordered in lexical order by their OutputID.

func (LexicalOrderedOutputs) Len added in v1.1.0 

func (l LexicalOrderedOutputs) Len() int

func (LexicalOrderedOutputs) Less added in v1.1.0 

func (l LexicalOrderedOutputs) Less(i, j int) bool

func (LexicalOrderedOutputs) Swap added in v1.1.0 

func (l LexicalOrderedOutputs) Swap(i, j int)

type MergeInfo 

type MergeInfo struct {
	// The header of the full snapshot.
	FullSnapshotHeader *ReadFileHeader
	// The header of the delta snapshot.
	DeltaSnapshotHeader *ReadFileHeader
	// The header of the merged snapshot.
	MergedSnapshotHeader *ReadFileHeader
}

MergeInfo holds information about a merge of a full and delta snapshot.

func MergeSnapshotsFiles 

func MergeSnapshotsFiles(fullPath string, deltaPath string, targetFileName string) (*MergeInfo, error)

MergeSnapshotsFiles merges the given full and delta snapshots to create an updated full snapshot. The result is a full snapshot file containing the ledger outputs corresponding to the snapshot index of the specified delta snapshot. The target file does not include any milestone diffs and the ledger and snapshot index are equal. This function consumes disk space over memory by importing the full snapshot into a temporary database, applying the delta diffs onto it and then writing out the merged state.

type MilestoneDiff 

type MilestoneDiff struct {
	// The milestone payload itself.
	Milestone *iotago.Milestone `json:"milestone"`
	// The created outputs with this milestone.
	Created []*Output `json:"created"`
	// The consumed spents with this milestone.
	Consumed []*Spent `json:"consumed"`
	// The consumed treasury output with this milestone.
	SpentTreasuryOutput *utxo.TreasuryOutput
}

MilestoneDiff represents the outputs which were created and consumed for the given milestone and the message itself which contains the milestone.

func (*MilestoneDiff) MarshalBinary 

func (md *MilestoneDiff) MarshalBinary() ([]byte, error)

func (*MilestoneDiff) TreasuryOutput 

func (md *MilestoneDiff) TreasuryOutput() *utxo.TreasuryOutput

TreasuryOutput extracts the new treasury output from within the milestone receipt. Might return nil if there is no receipt within the milestone.

type MilestoneDiffConsumerFunc 

type MilestoneDiffConsumerFunc func(milestoneDiff *MilestoneDiff) error

MilestoneDiffConsumerFunc consumes the given MilestoneDiff. A returned error signals to cancel further reading.

type MilestoneDiffProducerFunc 

type MilestoneDiffProducerFunc func() (*MilestoneDiff, error)

MilestoneDiffProducerFunc yields a milestone diff to be written to a snapshot or nil if no more is available.

type MilestoneRetrieverFunc 

type MilestoneRetrieverFunc func(index milestone.Index) (*iotago.Milestone, error)

MilestoneRetrieverFunc is a function which returns the milestone for the given index.

func MilestoneRetrieverFromStorage 

func MilestoneRetrieverFromStorage(dbStorage *storage.Storage) MilestoneRetrieverFunc

MilestoneRetrieverFromStorage creates a MilestoneRetrieverFunc which access the storage. If it can not retrieve a wanted milestone it panics.

type MsDiffDirection 

type MsDiffDirection byte

MsDiffDirection determines the milestone diff direction. 

const (
	// MsDiffDirectionBackwards defines to produce milestone diffs in backwards direction.
	MsDiffDirectionBackwards MsDiffDirection = iota
	// MsDiffDirectionOnwards defines to produce milestone diffs in onwards direction.
	MsDiffDirectionOnwards
)

type Output 

type Output struct {
	// The message ID of the message that contained the transaction where this output was created.
	MessageID [iotago.MessageIDLength]byte `json:"message_id"`
	// The transaction ID and the index of the output.
	OutputID [iotago.TransactionIDLength + 2]byte `json:"output_id"`
	// The type of the output.
	OutputType iotago.OutputType `json:"output_type"`
	// The underlying address to which this output deposits to.
	Address serializer.Serializable `json:"address"`
	// The amount of the deposit.
	Amount uint64 `json:"amount"`
}

Output defines an output within a snapshot.

func (*Output) MarshalBinary 

func (s *Output) MarshalBinary() ([]byte, error)

type OutputConsumerFunc 

type OutputConsumerFunc func(output *Output) error

OutputConsumerFunc consumes the given output. A returned error signals to cancel further reading.

type OutputProducerFunc 

type OutputProducerFunc func() (*Output, error)

OutputProducerFunc yields an output to be written to a snapshot or nil if no more is available.

type PruningMetrics 

type PruningMetrics struct {
	DurationPruneUnreferencedMessages    time.Duration
	DurationTraverseMilestoneCone        time.Duration
	DurationPruneMilestone               time.Duration
	DurationPruneMessages                time.Duration
	DurationSetSnapshotInfo              time.Duration
	DurationPruningMilestoneIndexChanged time.Duration
	DurationTotal                        time.Duration
}

PruningMetrics holds metrics about a database pruning run.

type ReadFileHeader 

type ReadFileHeader struct {
	FileHeader
	// The time at which the snapshot was taken.
	Timestamp uint64
	// The count of solid entry points.
	SEPCount uint64
	// The count of outputs. This count is zero if a delta snapshot has been read.
	OutputCount uint64
	// The count of milestone diffs.
	MilestoneDiffCount uint64
}

ReadFileHeader is a FileHeader but with additional content read from the snapshot.

func CreateSnapshotFromStorage added in v1.2.0 

func CreateSnapshotFromStorage(
	ctx context.Context,
	dbStorage *storage.Storage,
	utxoManager *utxo.Manager,
	filePath string,
	targetIndex milestone.Index,
	solidEntryPointCheckThresholdPast milestone.Index,
	solidEntryPointCheckThresholdFuture milestone.Index,
) (*ReadFileHeader, error)

CreateSnapshotFromStorage creates a snapshot file by streaming data from the database into a snapshot file.

func ReadSnapshotHeader 

func ReadSnapshotHeader(reader io.Reader) (*ReadFileHeader, error)

ReadSnapshotHeader reads the snapshot header from the given reader.

func ReadSnapshotHeaderFromFile added in v1.0.2 

func ReadSnapshotHeaderFromFile(filePath string) (*ReadFileHeader, error)

ReadSnapshotHeaderFromFile reads the header of the given snapshot file.

type SEPConsumerFunc 

type SEPConsumerFunc func(hornet.MessageID) error

SEPConsumerFunc consumes the given solid entry point. A returned error signals to cancel further reading.

type SEPProducerFunc 

type SEPProducerFunc func() (hornet.MessageID, error)

SEPProducerFunc yields a solid entry point to be written to a snapshot or nil if no more is available.

type SnapshotManager added in v1.1.0 

type SnapshotManager struct {
	// the logger used to log events.
	*utils.WrappedLogger

	Events *Events
	// contains filtered or unexported fields
}

SnapshotManager handles reading and writing snapshot data.

func NewSnapshotManager added in v1.1.0 

func NewSnapshotManager(
	log *logger.Logger,
	tangleDatabase *database.Database,
	utxoDatabase *database.Database,
	storage *storage.Storage,
	syncManager *syncmanager.SyncManager,
	utxoManager *utxo.Manager,
	networkID uint64,
	networkIDSource string,
	snapshotFullPath string,
	snapshotDeltaPath string,
	deltaSnapshotSizeThresholdPercentage float64,
	downloadTargets []*DownloadTarget,
	solidEntryPointCheckThresholdPast milestone.Index,
	solidEntryPointCheckThresholdFuture milestone.Index,
	additionalPruningThreshold milestone.Index,
	snapshotDepth milestone.Index,
	snapshotInterval milestone.Index,
	pruningMilestonesEnabled bool,
	pruningMilestonesMaxMilestonesToKeep milestone.Index,
	pruningSizeEnabled bool,
	pruningSizeTargetSizeBytes int64,
	pruningSizeThresholdPercentage float64,
	pruningSizeCooldownTime time.Duration,
	pruneReceipts bool) *SnapshotManager

NewSnapshotManager creates a new snapshot manager instance.

func (*SnapshotManager) CheckCurrentSnapshot added in v1.1.0 

func (s *SnapshotManager) CheckCurrentSnapshot(snapshotInfo *storage.SnapshotInfo) error

CheckCurrentSnapshot checks that the current snapshot info is valid regarding its network ID and the ledger state.

func (*SnapshotManager) CreateDeltaSnapshot added in v1.1.0 

func (s *SnapshotManager) CreateDeltaSnapshot(ctx context.Context, targetIndex milestone.Index, filePath string, writeToDatabase bool, snapshotFullPath ...string) error

CreateDeltaSnapshot creates a delta snapshot for the given target milestone index.

func (*SnapshotManager) CreateFullSnapshot added in v1.1.0 

func (s *SnapshotManager) CreateFullSnapshot(ctx context.Context, targetIndex milestone.Index, filePath string, writeToDatabase bool) error

CreateFullSnapshot creates a full snapshot for the given target milestone index.

func (*SnapshotManager) DownloadSnapshotFiles added in v1.1.0 

func (s *SnapshotManager) DownloadSnapshotFiles(ctx context.Context, wantedNetworkID uint64, fullPath string, deltaPath string, targets []*DownloadTarget) error

DownloadSnapshotFiles tries to download snapshots files from the given targets.

func (*SnapshotManager) HandleNewConfirmedMilestoneEvent added in v1.1.0 

func (s *SnapshotManager) HandleNewConfirmedMilestoneEvent(ctx context.Context, confirmedMilestoneIndex milestone.Index)

HandleNewConfirmedMilestoneEvent handles new confirmed milestone events which may trigger a delta snapshot creation and pruning.

func (*SnapshotManager) ImportSnapshots added in v1.1.0 

func (s *SnapshotManager) ImportSnapshots(ctx context.Context) error

ImportSnapshots imports snapshot data from the configured file paths. automatically downloads snapshot data if no files are available.

func (*SnapshotManager) IsSnapshottingOrPruning added in v1.1.0 

func (s *SnapshotManager) IsSnapshottingOrPruning() bool

func (*SnapshotManager) LoadSnapshotFromFile added in v1.1.0 

func (s *SnapshotManager) LoadSnapshotFromFile(ctx context.Context, snapshotType Type, networkID uint64, filePath string) (err error)

LoadSnapshotFromFile loads a snapshot file from the given file path into the storage.

func (*SnapshotManager) PruneDatabaseByDepth added in v1.1.0 

func (s *SnapshotManager) PruneDatabaseByDepth(ctx context.Context, depth milestone.Index) (milestone.Index, error)

func (*SnapshotManager) PruneDatabaseBySize added in v1.1.0 

func (s *SnapshotManager) PruneDatabaseBySize(ctx context.Context, targetSizeBytes int64) (milestone.Index, error)

func (*SnapshotManager) PruneDatabaseByTargetIndex added in v1.1.0 

func (s *SnapshotManager) PruneDatabaseByTargetIndex(ctx context.Context, targetIndex milestone.Index) (milestone.Index, error)

func (*SnapshotManager) SnapshotsFilesLedgerIndex added in v1.1.0 

func (s *SnapshotManager) SnapshotsFilesLedgerIndex() (milestone.Index, error)

SnapshotsFilesLedgerIndex returns the final ledger index if the snapshots from the configured file paths would be applied.

type SnapshotMetrics 

type SnapshotMetrics struct {
	DurationReadLockLedger                time.Duration
	DurationInit                          time.Duration
	DurationSetSnapshotInfo               time.Duration
	DurationSnapshotMilestoneIndexChanged time.Duration
	DurationHeader                        time.Duration
	DurationSolidEntryPoints              time.Duration
	DurationOutputs                       time.Duration
	DurationMilestoneDiffs                time.Duration
	DurationTotal                         time.Duration
}

SnapshotMetrics holds metrics about a snapshot creation run.

func StreamSnapshotDataTo 

func StreamSnapshotDataTo(writeSeeker io.WriteSeeker, timestamp uint64, header *FileHeader,
	sepProd SEPProducerFunc, outputProd OutputProducerFunc, msDiffProd MilestoneDiffProducerFunc) (*SnapshotMetrics, error)

StreamSnapshotDataTo streams a snapshot data into the given io.WriteSeeker. FileHeader.Type is used to determine whether to write a full or delta snapshot. If the type of the snapshot is Full, then OutputProducerFunc must be provided.

type Spent 

type Spent struct {
	Output
	// The transaction ID the funds were spent with.
	TargetTransactionID [iotago.TransactionIDLength]byte `json:"target_transaction_id"`
}

Spent defines a spent within a snapshot.

func (*Spent) MarshalBinary 

func (s *Spent) MarshalBinary() ([]byte, error)

type Type 

type Type byte

Type defines the type of the snapshot. 

const (
	// Full is a snapshot which contains the full ledger entry for a given milestone
	// plus the milestone diffs which subtracted to the ledger milestone reduce to the snapshot milestone ledger.
	Full Type = iota
	// Delta is a snapshot which contains solely diffs of milestones newer than a certain ledger milestone
	// instead of the complete ledger state of a given milestone.
	Delta
)

type UnspentTreasuryOutputConsumerFunc 

type UnspentTreasuryOutputConsumerFunc func(output *utxo.TreasuryOutput) error

UnspentTreasuryOutputConsumerFunc consumes the given treasury output. A returned error signals to cancel further reading.

type WriteCounter 

type WriteCounter struct {
	Expected uint64
	// contains filtered or unexported fields
}

WriteCounter counts the number of bytes written to it. It implements to the io.Writer interface and we can pass this into io.TeeReader() which will report progress on each write cycle.

func NewWriteCounter 

func NewWriteCounter(shutdownCtx context.Context, expected uint64) *WriteCounter

NewWriteCounter creates a new WriteCounter.

func (*WriteCounter) PrintProgress 

func (wc *WriteCounter) PrintProgress()

PrintProgress prints the current progress.

func (*WriteCounter) Write 

func (wc *WriteCounter) Write(p []byte) (int, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.