ledgerexporter

package
v0.0.0-...-a387ffb Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 26, 2024 License: Apache-2.0, Apache-2.0 Imports: 36 Imported by: 0

Documentation

Index

Constants

View Source 
const Pubnet = "pubnet"
View Source 
const Testnet = "testnet"

Variables

This section is empty.

Functions

func GetObjectKeyFromSequenceNumber 

func GetObjectKeyFromSequenceNumber(config ExporterConfig, ledgerSeq uint32) (string, error)

GetObjectKeyFromSequenceNumber generates the file name from the ledger sequence number based on configuration.

Types

type App 

type App struct {
	// contains filtered or unexported fields
}

func NewApp 

func NewApp() *App

func (*App) Run 

func (a *App) Run()

type Config 

type Config struct {
	AdminPort int `toml:"admin_port"`

	Network           string            `toml:"network"`
	DestinationURL    string            `toml:"destination_url"`
	ExporterConfig    ExporterConfig    `toml:"exporter_config"`
	StellarCoreConfig StellarCoreConfig `toml:"stellar_core_config"`

	//From command-line
	StartLedger          uint32 `toml:"start"`
	EndLedger            uint32 `toml:"end"`
	StartFromLastLedgers uint32 `toml:"from-last"`
}

func (*Config) GenerateCaptiveCoreConfig 

func (config *Config) GenerateCaptiveCoreConfig() ledgerbackend.CaptiveCoreConfig

func (*Config) LoadConfig 

func (config *Config) LoadConfig() error

type DataStore 

type DataStore interface {
	GetFile(ctx context.Context, path string) (io.ReadCloser, error)
	PutFile(ctx context.Context, path string, in io.WriterTo) error
	PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo) (bool, error)
	Exists(ctx context.Context, path string) (bool, error)
	Size(ctx context.Context, path string) (int64, error)
	Close() error
}

DataStore defines an interface for interacting with data storage

func NewDataStore 

func NewDataStore(ctx context.Context, destinationURL string) (DataStore, error)

NewDataStore creates a new DataStore based on the destination URL. Currently, only accepts GCS URLs.

type ExportManager 

type ExportManager struct {
	// contains filtered or unexported fields
}

ExportManager manages the creation and handling of export objects.

func NewExportManager 

func NewExportManager(config ExporterConfig, backend ledgerbackend.LedgerBackend, queue UploadQueue, prometheusRegistry *prometheus.Registry) *ExportManager

NewExportManager creates a new ExportManager with the provided configuration.

func (*ExportManager) AddLedgerCloseMeta 

func (e *ExportManager) AddLedgerCloseMeta(ctx context.Context, ledgerCloseMeta xdr.LedgerCloseMeta) error

AddLedgerCloseMeta adds ledger metadata to the current export object

func (*ExportManager) Run 

func (e *ExportManager) Run(ctx context.Context, startLedger, endLedger uint32) error

Run iterates over the specified range of ledgers, retrieves ledger data from the backend, and processes the corresponding ledger close metadata. The process continues until the ending ledger number is reached or a cancellation signal is received.

type ExporterConfig 

type ExporterConfig struct {
	LedgersPerFile    uint32 `toml:"ledgers_per_file"`
	FilesPerPartition uint32 `toml:"files_per_partition"`
}

type GCSDataStore 

type GCSDataStore struct {
	// contains filtered or unexported fields
}

GCSDataStore implements DataStore for GCS

func (GCSDataStore) Close 

func (b GCSDataStore) Close() error

Close closes the GCS client connection.

func (GCSDataStore) Exists 

func (b GCSDataStore) Exists(ctx context.Context, pth string) (bool, error)

Exists checks if a file exists in the GCS bucket.

func (GCSDataStore) GetFile 

func (b GCSDataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)

GetFile retrieves a file from the GCS bucket.

func (GCSDataStore) PutFile 

func (b GCSDataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo) error

PutFile uploads a file to GCS

func (GCSDataStore) PutFileIfNotExists 

func (b GCSDataStore) PutFileIfNotExists(ctx context.Context, filePath string, in io.WriterTo) (bool, error)

PutFileIfNotExists uploads a file to GCS only if it doesn't already exist.

func (GCSDataStore) Size 

func (b GCSDataStore) Size(ctx context.Context, pth string) (int64, error)

Size retrieves the size of a file in the GCS bucket.

type LedgerMetaArchive 

type LedgerMetaArchive struct {
	// contains filtered or unexported fields
}

LedgerMetaArchive represents a file with metadata and binary data.

func NewLedgerMetaArchive 

func NewLedgerMetaArchive(key string, startSeq uint32, endSeq uint32) *LedgerMetaArchive

NewLedgerMetaArchive creates a new LedgerMetaArchive instance.

func (*LedgerMetaArchive) AddLedger 

func (f *LedgerMetaArchive) AddLedger(ledgerCloseMeta xdr.LedgerCloseMeta) error

AddLedger adds a LedgerCloseMeta to the archive.

func (*LedgerMetaArchive) GetEndLedgerSequence 

func (f *LedgerMetaArchive) GetEndLedgerSequence() uint32

GetEndLedgerSequence returns the ending ledger sequence of the archive.

func (*LedgerMetaArchive) GetLedgerCount 

func (f *LedgerMetaArchive) GetLedgerCount() uint32

GetLedgerCount returns the number of ledgers currently in the archive.

func (*LedgerMetaArchive) GetObjectKey 

func (f *LedgerMetaArchive) GetObjectKey() string

GetObjectKey returns the object key of the archive.

func (*LedgerMetaArchive) GetStartLedgerSequence 

func (f *LedgerMetaArchive) GetStartLedgerSequence() uint32

GetStartLedgerSequence returns the starting ledger sequence of the archive.

type MockDataStore 

type MockDataStore struct {
	mock.Mock
}

MockDataStore is a mock implementation for the Storage interface.

func (*MockDataStore) Close 

func (m *MockDataStore) Close() error

func (*MockDataStore) Exists 

func (m *MockDataStore) Exists(ctx context.Context, path string) (bool, error)

func (*MockDataStore) GetFile 

func (m *MockDataStore) GetFile(ctx context.Context, path string) (io.ReadCloser, error)

func (*MockDataStore) PutFile 

func (m *MockDataStore) PutFile(ctx context.Context, path string, in io.WriterTo) error

func (*MockDataStore) PutFileIfNotExists 

func (m *MockDataStore) PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo) (bool, error)

func (*MockDataStore) Size 

func (m *MockDataStore) Size(ctx context.Context, path string) (int64, error)

type StellarCoreConfig 

type StellarCoreConfig struct {
	NetworkPassphrase     string   `toml:"network_passphrase"`
	HistoryArchiveUrls    []string `toml:"history_archive_urls"`
	StellarCoreBinaryPath string   `toml:"stellar_core_binary_path"`
	CaptiveCoreTomlPath   string   `toml:"captive_core_toml_path"`
}

type UploadQueue 

type UploadQueue struct {
	// contains filtered or unexported fields
}

UploadQueue is a queue of LedgerMetaArchive objects which are scheduled for upload

func NewUploadQueue 

func NewUploadQueue(size int, prometheusRegistry *prometheus.Registry) UploadQueue

NewUploadQueue constructs a new UploadQueue

func (UploadQueue) Close 

func (u UploadQueue) Close()

Close will close the queue.

func (UploadQueue) Dequeue 

func (u UploadQueue) Dequeue(ctx context.Context) (*LedgerMetaArchive, bool, error)

Dequeue will pop a task off the queue. Dequeue may block if the queue is empty.

func (UploadQueue) Enqueue 

func (u UploadQueue) Enqueue(ctx context.Context, archive *LedgerMetaArchive) error

Enqueue will add an upload task to the queue. Enqueue may block if the queue is full.

type Uploader 

type Uploader struct {
	// contains filtered or unexported fields
}

Uploader is responsible for uploading data to a storage destination.

func NewUploader 

func NewUploader(
	destination DataStore,
	queue UploadQueue,
	prometheusRegistry *prometheus.Registry,
) Uploader

NewUploader constructs a new Uploader instance

func (Uploader) Run 

func (u Uploader) Run(ctx context.Context) error

Run starts the uploader, continuously listening for LedgerMetaArchive objects to upload.

func (Uploader) Upload 

func (u Uploader) Upload(ctx context.Context, metaArchive *LedgerMetaArchive) error

Upload uploads the serialized binary data of ledger TxMeta to the specified destination.

type XDRGzipDecoder 

type XDRGzipDecoder struct {
	XdrPayload interface{}
}

func (*XDRGzipDecoder) ReadFrom 

func (d *XDRGzipDecoder) ReadFrom(r io.Reader) (int64, error)

type XDRGzipEncoder 

type XDRGzipEncoder struct {
	XdrPayload interface{}
}

func (*XDRGzipEncoder) WriteTo 

func (g *XDRGzipEncoder) WriteTo(w io.Writer) (int64, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.