datastore

Documentation

Overview

Package datastore manages chunk and snapshot files in object storage, including upload, download, listing, and revision parsing.

Index

• func ChunkKey(revision int64) string
• func Download(ctx context.Context, store storage.ObjectStorage, key string, size int64, ...) (io.ReadCloser, error)
• func DownloadAndImportFile(ctx context.Context, logger *slog.Logger, db localdb.Database, ...) error
• func MarshalChunk(record *pb.Record, nodeID string) (key string, data []byte, err error)
• func MarshalChunkBatch(records []*pb.Record, nodeID string) (key string, data []byte, err error)
• func SnapshotKey(revision int64) string
• func Upload(ctx context.Context, store storage.ObjectStorage, key, filePath string) error
• type FileInfo
  • func ListChunks(ctx context.Context, store storage.ObjectStorage, fromRevision int64) ([]FileInfo, error)
  • func ListChunksForCleanup(ctx context.Context, store storage.ObjectStorage, upToRevision int64) ([]FileInfo, error)
  • func ListSnapshots(ctx context.Context, store storage.ObjectStorage) ([]FileInfo, error)
• type LatestSnapshotInfo
  • func GetLatestSnapshot(ctx context.Context, store storage.ObjectStorage) (*LatestSnapshotInfo, error)

Functions

func ChunkKey

func ChunkKey(revision int64) string

ChunkKey generates the object storage key for a chunk file

func Download

func Download(ctx context.Context, store storage.ObjectStorage, key string, size int64, dataDir string, tempFiles *[]string) (io.ReadCloser, error)

Download retrieves an object from storage, using a temp file for large objects to avoid holding everything in memory.

func DownloadAndImportFile

func DownloadAndImportFile(
	ctx context.Context,
	logger *slog.Logger,
	db localdb.Database,
	store storage.ObjectStorage,
	dataDir string,
	key string,
	size int64,
	expectedKind pb.FileKind,
	storageMetrics *metrics.ObjectStorageMetrics,
) error

DownloadAndImportFile downloads a Netsy chunk or snapshot file from object storage and replays its records into SQLite.

func MarshalChunk

func MarshalChunk(record *pb.Record, nodeID string) (key string, data []byte, err error)

MarshalChunk serialises a single record into the chunk datafile format, returning the object-storage key and the raw bytes.

func MarshalChunkBatch

func MarshalChunkBatch(records []*pb.Record, nodeID string) (key string, data []byte, err error)

MarshalChunkBatch serialises multiple records into one chunk datafile with smart compression, keyed by the highest revision in the batch.

func SnapshotKey

func SnapshotKey(revision int64) string

SnapshotKey generates the object storage key for a snapshot file

func Upload

func Upload(ctx context.Context, store storage.ObjectStorage, key, filePath string) error

Upload uploads a local file to object storage at the given key

Types

type FileInfo

type FileInfo struct {
	Key      string
	Size     int64
	Revision int64
}

FileInfo represents a chunk or snapshot file with its parsed revision

func ListChunks

func ListChunks(ctx context.Context, store storage.ObjectStorage, fromRevision int64) ([]FileInfo, error)

ListChunks returns chunk files with revision > fromRevision, sorted oldest first

func ListChunksForCleanup

func ListChunksForCleanup(ctx context.Context, store storage.ObjectStorage, upToRevision int64) ([]FileInfo, error)

ListChunksForCleanup returns chunk files with revision <= upToRevision, sorted oldest first

func ListSnapshots

func ListSnapshots(ctx context.Context, store storage.ObjectStorage) ([]FileInfo, error)

ListSnapshots returns all snapshot files sorted by revision (newest first)

type LatestSnapshotInfo

type LatestSnapshotInfo struct {
	Revision int64
	Key      string
	Size     int64
	Found    bool
}

LatestSnapshotInfo contains information about the latest snapshot

func GetLatestSnapshot

func GetLatestSnapshot(ctx context.Context, store storage.ObjectStorage) (*LatestSnapshotInfo, error)

GetLatestSnapshot returns information about the latest snapshot, or Found=false if none exists