Documentation
¶
Index ¶
- Constants
- Variables
- func FindLatestLedgerSequence(ctx context.Context, datastore DataStore) (uint32, error)
- func FindLatestLedgerUpToSequence(ctx context.Context, datastore DataStore, end uint32, schema DataStoreSchema) (uint32, error)
- func FindOldestLedgerSequence(ctx context.Context, datastore DataStore, schema DataStoreSchema) (uint32, error)
- func GetLedgerFileExtension(ctx context.Context, dataStore DataStore) (string, error)
- type ConfigMismatchError
- type DataStore
- func FromGCSClient(ctx context.Context, client *storage.Client, bucketPath string) (DataStore, error)
- func FromS3Client(ctx context.Context, client *s3.Client, bucketPath string) (DataStore, error)
- func NewDataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error)
- func NewGCSDataStore(ctx context.Context, dataStoreConfig DataStoreConfig) (DataStore, error)
- func NewS3DataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error)
- type DataStoreConfig
- type DataStoreSchema
- type DatastoreManifest
- type GCSDataStore
- func (b GCSDataStore) Close() error
- func (b GCSDataStore) Exists(ctx context.Context, pth string) (bool, error)
- func (b GCSDataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)
- func (b GCSDataStore) GetFileAttrs(ctx context.Context, filePath string) (*storage.ObjectAttrs, error)
- func (b GCSDataStore) GetFileLastModified(ctx context.Context, filePath string) (time.Time, error)
- func (b GCSDataStore) GetFileMetadata(ctx context.Context, filePath string) (map[string]string, error)
- func (b GCSDataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)
- func (b GCSDataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo, ...) error
- func (b GCSDataStore) PutFileIfNotExists(ctx context.Context, filePath string, in io.WriterTo, ...) (bool, error)
- func (b GCSDataStore) Size(ctx context.Context, pth string) (int64, error)
- type ListFileOptions
- type MetaData
- type MockDataStore
- func (m *MockDataStore) Close() error
- func (m *MockDataStore) Exists(ctx context.Context, path string) (bool, error)
- func (m *MockDataStore) GetFile(ctx context.Context, path string) (io.ReadCloser, error)
- func (m *MockDataStore) GetFileLastModified(ctx context.Context, filePath string) (time.Time, error)
- func (m *MockDataStore) GetFileMetadata(ctx context.Context, path string) (map[string]string, error)
- func (m *MockDataStore) GetSchema() DataStoreSchema
- func (m *MockDataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)
- func (m *MockDataStore) PutFile(ctx context.Context, path string, in io.WriterTo, metadata map[string]string) error
- func (m *MockDataStore) PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo, metadata map[string]string) (bool, error)
- func (m *MockDataStore) Size(ctx context.Context, path string) (int64, error)
- type S3DataStore
- func (b S3DataStore) Close() error
- func (b S3DataStore) Exists(ctx context.Context, filePath string) (bool, error)
- func (b S3DataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)
- func (b S3DataStore) GetFileLastModified(ctx context.Context, filePath string) (time.Time, error)
- func (b S3DataStore) GetFileMetadata(ctx context.Context, filePath string) (map[string]string, error)
- func (b S3DataStore) HeadObject(ctx context.Context, filePath string) (*s3.HeadObjectOutput, error)
- func (b S3DataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)
- func (b S3DataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo, ...) error
- func (b S3DataStore) PutFileIfNotExists(ctx context.Context, filePath string, in io.WriterTo, ...) (bool, error)
- func (b S3DataStore) Size(ctx context.Context, filePath string) (int64, error)
Constants ¶
const (
Version = "1.0"
)
Variables ¶
var ErrNoLedgerFiles = errors.New("no ledger files found")
var ErrNoValidLedgerFiles = errors.New("no valid ledger files found on the data store")
Functions ¶
func FindLatestLedgerSequence ¶
FindLatestLedgerSequence returns the absolute latest ledger sequence number stored in the datastore.
func FindLatestLedgerUpToSequence ¶
func FindLatestLedgerUpToSequence(ctx context.Context, datastore DataStore, end uint32, schema DataStoreSchema) (uint32, error)
FindLatestLedgerUpToSequence finds the latest ledger sequence number that is less than or equal to a given 'end' sequence.
func FindOldestLedgerSequence ¶
func FindOldestLedgerSequence(ctx context.Context, datastore DataStore, schema DataStoreSchema) (uint32, error)
FindOldestLedgerSequence finds the oldest existing ledger in the datastore. It uses a binary search on the range of all known ledgers (from sequence 2 to the latest) to efficiently locate the first existing ledger file.
Types ¶
type ConfigMismatchError ¶
type ConfigMismatchError struct {
Diffs []string
}
func (*ConfigMismatchError) Error ¶
func (e *ConfigMismatchError) Error() string
type DataStore ¶
type DataStore interface { GetFileMetadata(ctx context.Context, path string) (map[string]string, error) GetFileLastModified(ctx context.Context, filePath string) (time.Time, error) GetFile(ctx context.Context, path string) (io.ReadCloser, error) PutFile(ctx context.Context, path string, in io.WriterTo, metaData map[string]string) error PutFileIfNotExists(ctx context.Context, path string, in io.WriterTo, metaData map[string]string) (bool, error) Exists(ctx context.Context, path string) (bool, error) Size(ctx context.Context, path string) (int64, error) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error) Close() error }
DataStore defines an interface for interacting with data storage
func FromGCSClient ¶
func FromS3Client ¶
func NewDataStore ¶
func NewDataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error)
NewDataStore factory, it creates a new DataStore based on the config type
func NewGCSDataStore ¶
func NewGCSDataStore(ctx context.Context, dataStoreConfig DataStoreConfig) (DataStore, error)
func NewS3DataStore ¶
func NewS3DataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataStore, error)
type DataStoreConfig ¶
type DataStoreConfig struct { Type string `toml:"type"` Params map[string]string `toml:"params"` Schema DataStoreSchema `toml:"schema"` NetworkPassphrase string Compression string }
DataStoreConfig defines user-provided configuration used to initialize a DataStore.
type DataStoreSchema ¶
type DataStoreSchema struct { LedgersPerFile uint32 `toml:"ledgers_per_file"` FilesPerPartition uint32 `toml:"files_per_partition"` FileExtension string // Optional – for backward (zstd) compatibility only }
func LoadSchema ¶
func LoadSchema(ctx context.Context, dataStore DataStore, cfg DataStoreConfig) (DataStoreSchema, error)
LoadSchema reads the datastore manifest from the given DataStore and returns its schema configuration.
func (DataStoreSchema) GetObjectKeyFromSequenceNumber ¶
func (ec DataStoreSchema) GetObjectKeyFromSequenceNumber(ledgerSeq uint32) string
GetObjectKeyFromSequenceNumber generates the object key name from the ledger sequence number based on configuration.
func (DataStoreSchema) GetSequenceNumberEndBoundary ¶
func (ec DataStoreSchema) GetSequenceNumberEndBoundary(ledgerSeq uint32) uint32
func (DataStoreSchema) GetSequenceNumberStartBoundary ¶
func (ec DataStoreSchema) GetSequenceNumberStartBoundary(ledgerSeq uint32) uint32
type DatastoreManifest ¶
type DatastoreManifest struct { NetworkPassphrase string `json:"networkPassphrase"` Version string `json:"version"` Compression string `json:"compression"` LedgersPerFile uint32 `json:"ledgersPerBatch"` FilesPerPartition uint32 `json:"batchesPerPartition"` }
DatastoreManifest represents the persisted configuration stored in the object store.
func PublishConfig ¶
func PublishConfig(ctx context.Context, dataStore DataStore, cfg DataStoreConfig) (DatastoreManifest, bool, error)
PublishConfig ensures that a datastore manifest exists and matches the provided configuration. If the manifest is missing, it creates one. Returns the manifest, whether it was created, and any error encountered.
type GCSDataStore ¶
type GCSDataStore struct {
// contains filtered or unexported fields
}
GCSDataStore implements DataStore for GCS
func (GCSDataStore) Close ¶
func (b GCSDataStore) Close() error
Close closes the GCS client connection.
func (GCSDataStore) GetFile ¶
func (b GCSDataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)
GetFile retrieves a file from the GCS bucket.
func (GCSDataStore) GetFileAttrs ¶
func (b GCSDataStore) GetFileAttrs(ctx context.Context, filePath string) (*storage.ObjectAttrs, error)
func (GCSDataStore) GetFileLastModified ¶
GetFileLastModified retrieves the last modified time of a file in the GCS bucket.
func (GCSDataStore) GetFileMetadata ¶
func (b GCSDataStore) GetFileMetadata(ctx context.Context, filePath string) (map[string]string, error)
GetFileMetadata retrieves the metadata for the specified file in the GCS bucket.
func (GCSDataStore) ListFilePaths ¶
func (b GCSDataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)
ListFilePaths lists up to 'limit' file paths under the provided prefix. Returned paths are relative to the bucket prefix. and ordered lexicographically ascending as provided by the backend. If limit <= 0, implementations default to a cap of 1,000; values > 1,000 are capped to 1,000.
func (GCSDataStore) PutFile ¶
func (b GCSDataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) error
PutFile uploads a file to GCS
type ListFileOptions ¶
type ListFileOptions struct { // Prefix filters the results to only include keys that start with this string. Prefix string // StartAfter specifies the key from which to begin listing. The returned keys will be // lexicographically greater than this value. StartAfter string // Limit restricts the number of keys returned. A value of 0 will use the default limit, // and any value above listFilePathsMaxLimit will be automatically capped. Limit uint32 }
ListFileOptions controls how ListFilePaths enumerates objects.
type MetaData ¶
type MockDataStore ¶
MockDataStore is a mock implementation for the Storage interface.
func (*MockDataStore) Close ¶
func (m *MockDataStore) Close() error
func (*MockDataStore) GetFile ¶
func (m *MockDataStore) GetFile(ctx context.Context, path string) (io.ReadCloser, error)
func (*MockDataStore) GetFileLastModified ¶
func (*MockDataStore) GetFileMetadata ¶
func (*MockDataStore) GetSchema ¶
func (m *MockDataStore) GetSchema() DataStoreSchema
func (*MockDataStore) ListFilePaths ¶
func (m *MockDataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)
func (*MockDataStore) PutFileIfNotExists ¶
type S3DataStore ¶
type S3DataStore struct {
// contains filtered or unexported fields
}
S3DataStore implements DataStore for AWS S3 and S3-compatible services.
func (S3DataStore) Close ¶
func (b S3DataStore) Close() error
Close does nothing for S3ObjectStore as it does not maintain a persistent connection.
func (S3DataStore) GetFile ¶
func (b S3DataStore) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)
GetFile retrieves a file from the S3-compatible bucket.
func (S3DataStore) GetFileLastModified ¶
GetFileLastModified retrieves the last modified time of a file in the S3-compatible bucket.
func (S3DataStore) GetFileMetadata ¶
func (b S3DataStore) GetFileMetadata(ctx context.Context, filePath string) (map[string]string, error)
GetFileMetadata retrieves the metadata for the specified file in the S3-compatible bucket.
func (S3DataStore) HeadObject ¶
func (b S3DataStore) HeadObject(ctx context.Context, filePath string) (*s3.HeadObjectOutput, error)
func (S3DataStore) ListFilePaths ¶
func (b S3DataStore) ListFilePaths(ctx context.Context, options ListFileOptions) ([]string, error)
ListFilePaths lists up to 'limit' file paths under the provided prefix. Returned paths are relative to the bucket prefix. and ordered lexicographically ascending as provided by the backend. If limit <= 0, implementations default to a cap of 1,000; values > 1,000 are capped to 1,000.
func (S3DataStore) PutFile ¶
func (b S3DataStore) PutFile(ctx context.Context, filePath string, in io.WriterTo, metaData map[string]string) error
PutFile uploads a file to S3-compatible bucket