datastore

package
v0.0.0-...-474a86a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 12, 2025 License: Apache-2.0, Apache-2.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	Version = "1.0"
)

Variables

View Source 
var ErrNoLedgerFiles = errors.New("no ledger files found")
View Source 
var ErrNoValidLedgerFiles = errors.New("no valid ledger files found on the data store")

Functions

func FindLatestLedgerSequence 

func FindLatestLedgerSequence(ctx context.Context, datastore DataStore) (uint32, error)

FindLatestLedgerSequence returns the absolute latest ledger sequence number stored in the datastore.

func FindLatestLedgerUpToSequence 

func FindLatestLedgerUpToSequence(ctx context.Context, datastore DataStore,
	end uint32, schema DataStoreSchema) (uint32, error)

FindLatestLedgerUpToSequence finds the latest ledger sequence number that is less than or equal to a given 'end' sequence.

func FindOldestLedgerSequence 

func FindOldestLedgerSequence(ctx context.Context, datastore DataStore, schema DataStoreSchema) (uint32, error)

FindOldestLedgerSequence finds the oldest existing ledger in the datastore. It uses a binary search on the range of all known ledgers (from sequence 2 to the latest) to efficiently locate the first existing ledger file.

func GetLedgerFileExtension 

func GetLedgerFileExtension(ctx context.Context, dataStore DataStore) (string, error)

Types

type ConfigMismatchError 

type ConfigMismatchError struct {
	Diffs []string
}

func (*ConfigMismatchError) Error 

func (e *ConfigMismatchError) Error() string

type DataStore 

type DataStore interface {
	GetFileMetadata(ctx context.Context, path string) (map[string]string, error)
	GetFileLastModified(ctx context.Context, filePath string) (time.Time, error)
	GetFile(ctx context.Context, path string) (io.ReadCloser, error)
	PutFile(ctx context.Context, path string, in io.WriterTo, metaData map[string]string) error
	PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo, metaData map[string]string) (bool, error)
	Exists(ctx context.Context, path string) (bool, error)
	Size(ctx context.Context, path string) (int64, error)
	ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)
	Close() error
}

DataStore defines an interface for interacting with data storage

func FromGCSClient 

func FromGCSClient(ctx context.Context, client *storage.Client, bucketPath string) (DataStore, error)

func FromS3Client 

func FromS3Client(ctx context.Context, client *s3.Client, bucketPath string) (DataStore, error)

func NewDataStore 

func NewDataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error)

NewDataStore factory, it creates a new DataStore based on the config type

func NewGCSDataStore 

func NewGCSDataStore(ctx context.Context, dataStoreConfig DataStoreConfig) (DataStore, error)

func NewS3DataStore 

func NewS3DataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error)

type DataStoreConfig 

type DataStoreConfig struct {
	Type              string            `toml:"type"`
	Params            map[string]string `toml:"params"`
	Schema            DataStoreSchema   `toml:"schema"`
	NetworkPassphrase string
	Compression       string
}

DataStoreConfig defines user-provided configuration used to initialize a DataStore.

type DataStoreSchema 

type DataStoreSchema struct {
	LedgersPerFile    uint32 `toml:"ledgers_per_file"`
	FilesPerPartition uint32 `toml:"files_per_partition"`
	FileExtension     string // Optional – for backward (zstd) compatibility only
}

func LoadSchema 

func LoadSchema(ctx context.Context, dataStore DataStore, cfg DataStoreConfig) (DataStoreSchema, error)

LoadSchema reads the datastore manifest from the given DataStore and returns its schema configuration.

func (DataStoreSchema) GetObjectKeyFromSequenceNumber 

func (ec DataStoreSchema) GetObjectKeyFromSequenceNumber(ledgerSeq uint32) string

GetObjectKeyFromSequenceNumber generates the object key name from the ledger sequence number based on configuration.

func (DataStoreSchema) GetSequenceNumberEndBoundary 

func (ec DataStoreSchema) GetSequenceNumberEndBoundary(ledgerSeq uint32) uint32

func (DataStoreSchema) GetSequenceNumberStartBoundary 

func (ec DataStoreSchema) GetSequenceNumberStartBoundary(ledgerSeq uint32) uint32

type DatastoreManifest 

type DatastoreManifest struct {
	NetworkPassphrase string `json:"networkPassphrase"`
	Version           string `json:"version"`
	Compression       string `json:"compression"`
	LedgersPerFile    uint32 `json:"ledgersPerBatch"`
	FilesPerPartition uint32 `json:"batchesPerPartition"`
}

DatastoreManifest represents the persisted configuration stored in the object store.

func PublishConfig 

func PublishConfig(ctx context.Context, dataStore DataStore, cfg DataStoreConfig) (DatastoreManifest, bool, error)

PublishConfig ensures that a datastore manifest exists and matches the provided configuration. If the manifest is missing, it creates one. Returns the manifest, whether it was created, and any error encountered.

type GCSDataStore 

type GCSDataStore struct {
	// contains filtered or unexported fields
}

GCSDataStore implements DataStore for GCS

func (GCSDataStore) Close 

func (b GCSDataStore) Close() error

Close closes the GCS client connection.

func (GCSDataStore) Exists 

func (b GCSDataStore) Exists(ctx context.Context, pth string) (bool, error)

Exists checks if a file exists in the GCS bucket.

func (GCSDataStore) GetFile 

func (b GCSDataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)

GetFile retrieves a file from the GCS bucket.

func (GCSDataStore) GetFileAttrs 

func (b GCSDataStore) GetFileAttrs(ctx context.Context, filePath string) (*storage.ObjectAttrs, error)

func (GCSDataStore) GetFileLastModified 

func (b GCSDataStore) GetFileLastModified(ctx context.Context, filePath string) (time.Time, error)

GetFileLastModified retrieves the last modified time of a file in the GCS bucket.

func (GCSDataStore) GetFileMetadata 

func (b GCSDataStore) GetFileMetadata(ctx context.Context, filePath string) (map[string]string, error)

GetFileMetadata retrieves the metadata for the specified file in the GCS bucket.

func (GCSDataStore) ListFilePaths 

func (b GCSDataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)

ListFilePaths lists up to 'limit' file paths under the provided prefix. Returned paths are relative to the bucket prefix. and ordered lexicographically ascending as provided by the backend. If limit <= 0, implementations default to a cap of 1,000; values > 1,000 are capped to 1,000.

func (GCSDataStore) PutFile 

func (b GCSDataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) error

PutFile uploads a file to GCS

func (GCSDataStore) PutFileIfNotExists 

func (b GCSDataStore) PutFileIfNotExists(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) (bool, error)

PutFileIfNotExists uploads a file to GCS only if it doesn't already exist.

func (GCSDataStore) Size 

func (b GCSDataStore) Size(ctx context.Context, pth string) (int64, error)

Size retrieves the size of a file in the GCS bucket.

type ListFileOptions 

type ListFileOptions struct {
	// Prefix filters the results to only include keys that start with this string.
	Prefix string

	// StartAfter specifies the key from which to begin listing. The returned keys will be
	// lexicographically greater than this value.
	StartAfter string

	// Limit restricts the number of keys returned. A value of 0 will use the default limit,
	// and any value above listFilePathsMaxLimit will be automatically capped.
	Limit uint32
}

ListFileOptions controls how ListFilePaths enumerates objects.

type MetaData 

type MetaData struct {
	StartLedger          uint32
	EndLedger            uint32
	StartLedgerCloseTime int64
	EndLedgerCloseTime   int64
	ProtocolVersion      uint32
	CoreVersion          string
	NetworkPassPhrase    string
	CompressionType      string
	Version              string
}

func NewMetaDataFromMap 

func NewMetaDataFromMap(data map[string]string) (MetaData, error)

func (MetaData) ToMap 

func (m MetaData) ToMap() map[string]string

type MockDataStore 

type MockDataStore struct {
	mock.Mock
}

MockDataStore is a mock implementation for the Storage interface.

func (*MockDataStore) Close 

func (m *MockDataStore) Close() error

func (*MockDataStore) Exists 

func (m *MockDataStore) Exists(ctx context.Context, path string) (bool, error)

func (*MockDataStore) GetFile 

func (m *MockDataStore) GetFile(ctx context.Context, path string) (io.ReadCloser, error)

func (*MockDataStore) GetFileLastModified 

func (m *MockDataStore) GetFileLastModified(ctx context.Context, filePath string) (time.Time, error)

func (*MockDataStore) GetFileMetadata 

func (m *MockDataStore) GetFileMetadata(ctx context.Context, path string) (map[string]string, error)

func (*MockDataStore) GetSchema 

func (m *MockDataStore) GetSchema() DataStoreSchema

func (*MockDataStore) ListFilePaths 

func (m *MockDataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)

func (*MockDataStore) PutFile 

func (m *MockDataStore) PutFile(ctx context.Context, path string, in io.WriterTo, metadata map[string]string) error

func (*MockDataStore) PutFileIfNotExists 

func (m *MockDataStore) PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo, metadata map[string]string) (bool, error)

func (*MockDataStore) Size 

func (m *MockDataStore) Size(ctx context.Context, path string) (int64, error)

type S3DataStore 

type S3DataStore struct {
	// contains filtered or unexported fields
}

S3DataStore implements DataStore for AWS S3 and S3-compatible services.

func (S3DataStore) Close 

func (b S3DataStore) Close() error

Close does nothing for S3ObjectStore as it does not maintain a persistent connection.

func (S3DataStore) Exists 

func (b S3DataStore) Exists(ctx context.Context, filePath string) (bool, error)

Exists checks if a file exists in the S3-compatible bucket.

func (S3DataStore) GetFile 

func (b S3DataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)

GetFile retrieves a file from the S3-compatible bucket.

func (S3DataStore) GetFileLastModified 

func (b S3DataStore) GetFileLastModified(ctx context.Context, filePath string) (time.Time, error)

GetFileLastModified retrieves the last modified time of a file in the S3-compatible bucket.

func (S3DataStore) GetFileMetadata 

func (b S3DataStore) GetFileMetadata(ctx context.Context, filePath string) (map[string]string, error)

GetFileMetadata retrieves the metadata for the specified file in the S3-compatible bucket.

func (S3DataStore) HeadObject 

func (b S3DataStore) HeadObject(ctx context.Context, filePath string) (*s3.HeadObjectOutput, error)

func (S3DataStore) ListFilePaths 

func (b S3DataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)

ListFilePaths lists up to 'limit' file paths under the provided prefix. Returned paths are relative to the bucket prefix. and ordered lexicographically ascending as provided by the backend. If limit <= 0, implementations default to a cap of 1,000; values > 1,000 are capped to 1,000.

func (S3DataStore) PutFile 

func (b S3DataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) error

PutFile uploads a file to S3-compatible bucket

func (S3DataStore) PutFileIfNotExists 

func (b S3DataStore) PutFileIfNotExists(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) (bool, error)

PutFileIfNotExists uploads a file to S3-compatible bucket only if it doesn't already exist.

func (S3DataStore) Size 

func (b S3DataStore) Size(ctx context.Context, filePath string) (int64, error)

Size retrieves the size of a file in the S3-compatible bucket.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.