store

Documentation

Index

• Variables
• type Config
• type FileStore
  • func NewFileStore(cfg Config, manager *backend.Manager) (*FileStore, error)
  • func (fs *FileStore) AddBackend(b *types.Backend)
  • func (fs *FileStore) Close() error
  • func (fs *FileStore) DeleteChunk(ctx context.Context, id types.ChunkID) error
  • func (fs *FileStore) DeleteObject(ctx context.Context, id uuid.UUID) error
  • func (fs *FileStore) GetBackend(id string) (*types.Backend, bool)
  • func (fs *FileStore) GetBackendStorage(id string) (types.BackendStorage, bool)
  • func (fs *FileStore) GetChunk(ctx context.Context, id types.ChunkID) (io.ReadCloser, error)
  • func (fs *FileStore) GetChunkData(ctx context.Context, id types.ChunkID) ([]byte, error)
  • func (fs *FileStore) GetChunkDataRange(ctx context.Context, id types.ChunkID, offset, length int64) ([]byte, error)
  • func (fs *FileStore) GetChunkInfo(id types.ChunkID) (*types.Chunk, error)
  • func (fs *FileStore) GetChunkRange(ctx context.Context, id types.ChunkID, offset, length int64) (io.ReadCloser, error)
  • func (fs *FileStore) GetDefaultBackend() *types.Backend
  • func (fs *FileStore) GetECGroup(id uuid.UUID) (*types.ECGroup, error)
  • func (fs *FileStore) GetIndexStats() (*IndexStats, error)
  • func (fs *FileStore) GetObject(ctx context.Context, id uuid.UUID) (*types.ObjectRef, error)
  • func (fs *FileStore) GetObjectData(ctx context.Context, id uuid.UUID) (io.ReadCloser, error)
  • func (fs *FileStore) GetObjectRange(ctx context.Context, id uuid.UUID, offset, length int64) (io.ReadCloser, error)
  • func (fs *FileStore) IterateChunks(fn func(id types.ChunkID, chunk types.Chunk) error) error
  • func (fs *FileStore) ListBackends() map[string]*types.Backend
  • func (fs *FileStore) PutObject(ctx context.Context, obj *types.ObjectRef, reader io.Reader) error
  • func (fs *FileStore) PutObjectWithCompression(ctx context.Context, obj *types.ObjectRef, reader io.Reader, ...) error
  • func (fs *FileStore) RemoveBackend(id string)
  • func (fs *FileStore) WriteChunk(ctx context.Context, chunkID types.ChunkID, data []byte, backendID string) (*types.ChunkRef, error)
• type IndexKind
• type IndexStats
• type Store

Variables

var (
	// ChunkTotalCount tracks the total number of chunks in the index
	ChunkTotalCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunks_total",
		Help:      "Total number of chunks in the index",
	})

	// ChunkTotalBytes tracks the total bytes across all chunks
	ChunkTotalBytes = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunks_bytes_total",
		Help:      "Total bytes across all chunks",
	})

	// ChunkOperations tracks chunk operations by type
	ChunkOperations = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunk_operations_total",
		Help:      "Total number of chunk operations",
	}, []string{"operation"}) // operation: "create", "deduplicate", "delete"

	// ChunkDedupeHits tracks successful deduplication hits
	ChunkDedupeHits = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunk_dedupe_hits_total",
		Help:      "Number of times a chunk was deduplicated instead of stored",
	})
)

Types

type Config

type Config struct {
	IndexPath string
	IndexKind IndexKind // memory or leveldb
	Backends  []*types.Backend
	ECScheme  types.ECScheme
}

Config holds FileStore configuration

type FileStore

type FileStore struct {
	// contains filtered or unexported fields
}

FileStore implements Store using indexed chunks on multiple backends

func NewFileStore

func NewFileStore(cfg Config, manager *backend.Manager) (*FileStore, error)

NewFileStore creates a new FileStore

func (*FileStore) AddBackend

func (fs *FileStore) AddBackend(b *types.Backend)

AddBackend registers a new backend

func (*FileStore) Close

func (fs *FileStore) Close() error

func (*FileStore) DeleteChunk

func (fs *FileStore) DeleteChunk(ctx context.Context, id types.ChunkID) error

DeleteChunk removes a chunk from the index. Note: This does NOT delete the chunk from backend storage. Use with GetBackendStorage().Delete() to fully remove a chunk.

func (*FileStore) DeleteObject

func (fs *FileStore) DeleteObject(ctx context.Context, id uuid.UUID) error

DeleteObject marks an object as deleted

func (*FileStore) GetBackend

func (fs *FileStore) GetBackend(id string) (*types.Backend, bool)

GetBackend returns a backend by ID

func (*FileStore) GetBackendStorage

func (fs *FileStore) GetBackendStorage(id string) (types.BackendStorage, bool)

GetBackendStorage returns the storage interface for a backend

func (*FileStore) GetChunk

func (fs *FileStore) GetChunk(ctx context.Context, id types.ChunkID) (io.ReadCloser, error)

GetChunk reads a chunk from storage, decompressing if needed. Returns the original (uncompressed) data.

func (*FileStore) GetChunkData

func (fs *FileStore) GetChunkData(ctx context.Context, id types.ChunkID) ([]byte, error)

GetChunkData reads a chunk and returns the decompressed data as bytes.

func (*FileStore) GetChunkDataRange

func (fs *FileStore) GetChunkDataRange(ctx context.Context, id types.ChunkID, offset, length int64) ([]byte, error)

GetChunkDataRange reads a range from a chunk and returns the data as bytes.

func (*FileStore) GetChunkInfo

func (fs *FileStore) GetChunkInfo(id types.ChunkID) (*types.Chunk, error)

GetChunkInfo returns chunk metadata by ID.

func (*FileStore) GetChunkRange

func (fs *FileStore) GetChunkRange(ctx context.Context, id types.ChunkID, offset, length int64) (io.ReadCloser, error)

GetChunkRange reads a range from a chunk, decompressing if needed. For compressed chunks, the entire chunk must be decompressed first.

func (*FileStore) GetDefaultBackend

func (fs *FileStore) GetDefaultBackend() *types.Backend

GetDefaultBackend returns the first available backend

func (*FileStore) GetECGroup

func (fs *FileStore) GetECGroup(id uuid.UUID) (*types.ECGroup, error)

GetECGroup returns EC group metadata by ID (for admin/debugging)

func (*FileStore) GetIndexStats

func (fs *FileStore) GetIndexStats() (*IndexStats, error)

GetIndexStats returns statistics about the chunk index. Uses Prometheus metrics for O(1) performance instead of iterating.

func (*FileStore) GetObject

func (fs *FileStore) GetObject(ctx context.Context, id uuid.UUID) (*types.ObjectRef, error)

GetObject retrieves object metadata by ID.

func (*FileStore) GetObjectData

func (fs *FileStore) GetObjectData(ctx context.Context, id uuid.UUID) (io.ReadCloser, error)

GetObjectData retrieves the full object data

func (*FileStore) GetObjectRange

func (fs *FileStore) GetObjectRange(ctx context.Context, id uuid.UUID, offset, length int64) (io.ReadCloser, error)

GetObjectRange retrieves a range of object data

func (*FileStore) IterateChunks

func (fs *FileStore) IterateChunks(fn func(id types.ChunkID, chunk types.Chunk) error) error

IterateChunks iterates over all chunks in the index. Used by reconciliation to compare local chunks with expected chunks.

func (*FileStore) ListBackends

func (fs *FileStore) ListBackends() map[string]*types.Backend

ListBackends returns all registered backend IDs

func (*FileStore) PutObject

func (fs *FileStore) PutObject(ctx context.Context, obj *types.ObjectRef, reader io.Reader) error

PutObject stores an object, chunking the data and writing to backends without compression.

func (*FileStore) PutObjectWithCompression

func (fs *FileStore) PutObjectWithCompression(ctx context.Context, obj *types.ObjectRef, reader io.Reader, algo compression.Algorithm) error

PutObjectWithCompression stores an object, chunking the data and writing to backends with optional compression. Each chunk is compressed independently using the specified algorithm.

func (*FileStore) RemoveBackend

func (fs *FileStore) RemoveBackend(id string)

RemoveBackend removes a backend

func (*FileStore) WriteChunk

func (fs *FileStore) WriteChunk(ctx context.Context, chunkID types.ChunkID, data []byte, backendID string) (*types.ChunkRef, error)

WriteChunk writes a chunk directly to storage (no compression). Used by migration to receive chunks from peer servers. The chunkID is provided by the caller (already known from source).

type IndexKind

type IndexKind string

IndexKind specifies the index backend type

const (
	IndexKindMemory  IndexKind = "memory"
	IndexKindLevelDB IndexKind = "leveldb"
)

type IndexStats

type IndexStats struct {
	TotalChunks int64 `json:"total_chunks"`
	TotalBytes  int64 `json:"total_bytes"`
}

IndexStats holds statistics about the chunk index

type Store

type Store interface {
	io.Closer

	// Object operations
	GetObject(ctx context.Context, id uuid.UUID) (*types.ObjectRef, error)
	PutObject(ctx context.Context, obj *types.ObjectRef, reader io.Reader) error
	DeleteObject(ctx context.Context, id uuid.UUID) error

	// Chunk operations
	GetChunk(ctx context.Context, id types.ChunkID) (io.ReadCloser, error)
	GetChunkRange(ctx context.Context, id types.ChunkID, offset, length int64) (io.ReadCloser, error)
}

Store handles chunk and object storage