storage

Documentation

Index

• Constants
• func ValidateObjectManifest(store Storage, manifest *pb.ObjectManifest) error
• type FilesystemStorage
  • func NewFilesystemStorage(dir string, opts *FilesystemStorageOpts) (*FilesystemStorage, error)
  • func (s *FilesystemStorage) GetChunk(id contenthash.Hash) (io.ReadCloser, error)
  • func (s *FilesystemStorage) GetObject(id contenthash.Hash) (*pb.ObjectManifest, error)
  • func (s *FilesystemStorage) PutChunk(id contenthash.Hash, r io.ReadSeeker) error
  • func (s *FilesystemStorage) PutObject(manifest *pb.ObjectManifest) error
• type FilesystemStorageOpts
• type InMemoryStorage
  • func NewInMemoryStorage(opts *InMemoryStorageOpts) (*InMemoryStorage, error)
  • func (s *InMemoryStorage) GetChunk(id contenthash.Hash) (io.ReadCloser, error)
  • func (s *InMemoryStorage) GetObject(id contenthash.Hash) (*pb.ObjectManifest, error)
  • func (s *InMemoryStorage) PutChunk(id contenthash.Hash, r io.ReadSeeker) error
  • func (s *InMemoryStorage) PutObject(manifest *pb.ObjectManifest) error
• type InMemoryStorageOpts
• type Storage
• type TieredStorage
  • func NewTieredStorage(stores ...Storage) *TieredStorage
  • func (s *TieredStorage) GetChunk(id contenthash.Hash) (io.ReadCloser, error)
  • func (s *TieredStorage) GetObject(id contenthash.Hash) (*pb.ObjectManifest, error)
  • func (s *TieredStorage) PutChunk(id contenthash.Hash, r io.ReadSeeker) error
  • func (s *TieredStorage) PutObject(manifest *pb.ObjectManifest) error

Constants

const (
	// DefaultMaxObjects is the default maximum number of objects to store in memory.
	DefaultMaxObjects = 10000
)

Functions

func ValidateObjectManifest

func ValidateObjectManifest(store Storage, manifest *pb.ObjectManifest) error

ValidateObjectManifest checks that the object manifest is valid.

Types

type FilesystemStorage

type FilesystemStorage struct {
	// contains filtered or unexported fields
}

FilesystemStorage is a storage backend that stores chunks and objects in the local filesystem.

func NewFilesystemStorage

func NewFilesystemStorage(dir string, opts *FilesystemStorageOpts) (*FilesystemStorage, error)

NewFilesystemStorage creates a new filesystem storage backend.

func (*FilesystemStorage) GetChunk

func (s *FilesystemStorage) GetChunk(id contenthash.Hash) (io.ReadCloser, error)

GetChunk retrieves a chunk from storage.

func (*FilesystemStorage) GetObject

func (s *FilesystemStorage) GetObject(id contenthash.Hash) (*pb.ObjectManifest, error)

GetObject retrieves an object manifest from storage.

func (*FilesystemStorage) PutChunk

func (s *FilesystemStorage) PutChunk(id contenthash.Hash, r io.ReadSeeker) error

PutChunk adds a chunk to the storage.

func (*FilesystemStorage) PutObject

func (s *FilesystemStorage) PutObject(manifest *pb.ObjectManifest) error

PutObject adds an object manifest to the storage.

type FilesystemStorageOpts

type FilesystemStorageOpts struct {
	// NoSync disables fsync() calls after writing files. This will increase
	// performance but may lead to data loss in the event of a system crash.
	NoSync bool
}

FilesystemStorageOpts is used to configure the behaviour of the filesystem storage backend.

type InMemoryStorage

type InMemoryStorage struct {
	// contains filtered or unexported fields
}

InMemoryStorage is an in-memory storage backend.

func NewInMemoryStorage

func NewInMemoryStorage(opts *InMemoryStorageOpts) (*InMemoryStorage, error)

NewInMemoryStorage creates a new in-memory storage backend.

func (*InMemoryStorage) GetChunk

func (s *InMemoryStorage) GetChunk(id contenthash.Hash) (io.ReadCloser, error)

GetChunk retrieves a chunk from storage.

func (*InMemoryStorage) GetObject

func (s *InMemoryStorage) GetObject(id contenthash.Hash) (*pb.ObjectManifest, error)

GetObject retrieves an object manifest from storage.

func (*InMemoryStorage) PutChunk

func (s *InMemoryStorage) PutChunk(id contenthash.Hash, r io.ReadSeeker) error

PutChunk adds a chunk to the storage.

func (*InMemoryStorage) PutObject

func (s *InMemoryStorage) PutObject(manifest *pb.ObjectManifest) error

PutObject adds an object manifest to the storage.

type InMemoryStorageOpts

type InMemoryStorageOpts struct {
	// MaxObjects is the maximum number of objects to store.
	MaxObjects int64
	// ChunkMemoryLimit is the maximum amount of memory to use for storing chunks.
	ChunkMemoryLimit int64
}

InMemoryStorageOpts is used to configure the behavior of the in-memory storage.

type Storage

type Storage interface {
	// GetObject retrieves an object manifest from storage.
	GetObject(id contenthash.Hash) (*pb.ObjectManifest, error)
	// PutObject adds an object manifest to the storage.
	PutObject(manifest *pb.ObjectManifest) error
	// GetChunk retrieves a chunk from storage.
	GetChunk(id contenthash.Hash) (io.ReadCloser, error)
	// PutChunk adds a chunk to the storage.
	PutChunk(id contenthash.Hash, r io.ReadSeeker) error
}

type TieredStorage

type TieredStorage struct {
	// contains filtered or unexported fields
}

TieredStorage is a storage that delegates to multiple other storage backends. It is useful for implementing read through caches.

func NewTieredStorage

func NewTieredStorage(stores ...Storage) *TieredStorage

NewTieredStorage creates a new tiered storage. The backends are ordered in descending order by read priority.

func (*TieredStorage) GetChunk

func (s *TieredStorage) GetChunk(id contenthash.Hash) (io.ReadCloser, error)

GetChunk retrieves a chunk from storage.

func (*TieredStorage) GetObject

func (s *TieredStorage) GetObject(id contenthash.Hash) (*pb.ObjectManifest, error)

GetObject retrieves an object manifest from storage.

func (*TieredStorage) PutChunk

func (s *TieredStorage) PutChunk(id contenthash.Hash, r io.ReadSeeker) error

PutChunk adds a chunk to the storage.

func (*TieredStorage) PutObject

func (s *TieredStorage) PutObject(manifest *pb.ObjectManifest) error

PutObject adds an object manifest to the storage.