store

Documentation

Index

• Variables
• type BlobStore
• type Blocklister
• type CachingBlobStore
  • func NewCachingBlobStore(origin, cache BlobStore) *CachingBlobStore
  • func (c *CachingBlobStore) Delete(hash string) error
  • func (c *CachingBlobStore) Get(hash string) (stream.Blob, error)
  • func (c *CachingBlobStore) Has(hash string) (bool, error)
  • func (c *CachingBlobStore) Put(hash string, blob stream.Blob) error
  • func (c *CachingBlobStore) PutSD(hash string, blob stream.Blob) error
• type DBBackedStore
  • func NewDBBackedStore(blobs BlobStore, db *db.SQL) *DBBackedStore
  • func (d *DBBackedStore) Block(hash string) error
  • func (d *DBBackedStore) Delete(hash string) error
  • func (d *DBBackedStore) Get(hash string) (stream.Blob, error)
  • func (d *DBBackedStore) Has(hash string) (bool, error)
  • func (d *DBBackedStore) MissingBlobsForKnownStream(sdHash string) ([]string, error)
  • func (d *DBBackedStore) Put(hash string, blob stream.Blob) error
  • func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error
  • func (d *DBBackedStore) Wants(hash string) (bool, error)
• type DiskBlobStore
  • func NewDiskBlobStore(dir string, prefixLength int) *DiskBlobStore
  • func (d *DiskBlobStore) Delete(hash string) error
  • func (d *DiskBlobStore) Get(hash string) (stream.Blob, error)
  • func (d *DiskBlobStore) Has(hash string) (bool, error)
  • func (d *DiskBlobStore) Put(hash string, blob stream.Blob) error
  • func (d *DiskBlobStore) PutSD(hash string, blob stream.Blob) error
  • func (d *DiskBlobStore) WipeOldestBlobs() (err error)
• type MemoryBlobStore
  • func NewMemoryBlobStore() *MemoryBlobStore
  • func (m *MemoryBlobStore) Debug() map[string]stream.Blob
  • func (m *MemoryBlobStore) Delete(hash string) error
  • func (m *MemoryBlobStore) Get(hash string) (stream.Blob, error)
  • func (m *MemoryBlobStore) Has(hash string) (bool, error)
  • func (m *MemoryBlobStore) Put(hash string, blob stream.Blob) error
  • func (m *MemoryBlobStore) PutSD(hash string, blob stream.Blob) error
• type S3BlobStore
  • func NewS3BlobStore(awsID, awsSecret, region, bucket string) *S3BlobStore
  • func (s *S3BlobStore) Delete(hash string) error
  • func (s *S3BlobStore) Get(hash string) (stream.Blob, error)
  • func (s *S3BlobStore) Has(hash string) (bool, error)
  • func (s *S3BlobStore) Put(hash string, blob stream.Blob) error
  • func (s *S3BlobStore) PutSD(hash string, blob stream.Blob) error

Variables

var ErrBlobNotFound = errors.Base("blob not found")

ErrBlobNotFound is a standard error when a blob is not found in the store.

Types

type BlobStore

type BlobStore interface {
	// Does blob exist in the store
	Has(hash string) (bool, error)
	// Get the blob from the store
	Get(hash string) (stream.Blob, error)
	// Put the blob into the store
	Put(hash string, blob stream.Blob) error
	// Put an SD blob into the store
	PutSD(hash string, blob stream.Blob) error
	// Delete the blob from the store
	Delete(hash string) error
}

BlobStore is an interface for handling blob storage.

type Blocklister

type Blocklister interface {
	// Block deletes the blob and prevents it from being uploaded in the future
	Block(hash string) error
	// Wants returns false if the hash exists in store or is blocked, true otherwise
	Wants(hash string) (bool, error)
}

Blocklister is a store that supports blocking blobs to prevent their inclusion in the store.

type CachingBlobStore

type CachingBlobStore struct {
	// contains filtered or unexported fields
}

CachingBlobStore combines two stores, typically a local and a remote store, to improve performance. Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.

func NewCachingBlobStore

func NewCachingBlobStore(origin, cache BlobStore) *CachingBlobStore

NewCachingBlobStore makes a new caching disk store and returns a pointer to it.

func (*CachingBlobStore) Delete

func (c *CachingBlobStore) Delete(hash string) error

Delete deletes the blob from the origin and the cache

func (*CachingBlobStore) Get

func (c *CachingBlobStore) Get(hash string) (stream.Blob, error)

Get tries to get the blob from the cache first, falling back to the origin. If the blob comes from the origin, it is also stored in the cache.

func (*CachingBlobStore) Has

func (c *CachingBlobStore) Has(hash string) (bool, error)

Has checks the cache and then the origin for a hash. It returns true if either store has it.

func (*CachingBlobStore) Put

func (c *CachingBlobStore) Put(hash string, blob stream.Blob) error

Put stores the blob in the origin and the cache

func (*CachingBlobStore) PutSD

func (c *CachingBlobStore) PutSD(hash string, blob stream.Blob) error

PutSD stores the sd blob in the origin and the cache

type DBBackedStore

type DBBackedStore struct {
	// contains filtered or unexported fields
}

DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.

func NewDBBackedStore

func NewDBBackedStore(blobs BlobStore, db *db.SQL) *DBBackedStore

NewDBBackedStore returns an initialized store pointer.

func (*DBBackedStore) Block

func (d *DBBackedStore) Block(hash string) error

Block deletes the blob and prevents it from being uploaded in the future

func (*DBBackedStore) Delete

func (d *DBBackedStore) Delete(hash string) error

func (*DBBackedStore) Get

func (d *DBBackedStore) Get(hash string) (stream.Blob, error)

Get gets the blob

func (*DBBackedStore) Has

func (d *DBBackedStore) Has(hash string) (bool, error)

Has returns true if the blob is in the store

func (*DBBackedStore) MissingBlobsForKnownStream

func (d *DBBackedStore) MissingBlobsForKnownStream(sdHash string) ([]string, error)

MissingBlobsForKnownStream returns missing blobs for an existing stream WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks like no blobs are missing

func (*DBBackedStore) Put

func (d *DBBackedStore) Put(hash string, blob stream.Blob) error

Put stores the blob in the S3 store and stores the blob information in the DB.

func (*DBBackedStore) PutSD

func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error

PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if there is an error storing the blob information in the DB.

func (*DBBackedStore) Wants

func (d *DBBackedStore) Wants(hash string) (bool, error)

Wants returns false if the hash exists or is blocked, true otherwise

type DiskBlobStore

type DiskBlobStore struct {
	// contains filtered or unexported fields
}

DiskBlobStore stores blobs on a local disk

func NewDiskBlobStore

func NewDiskBlobStore(dir string, prefixLength int) *DiskBlobStore

NewDiskBlobStore returns an initialized file disk store pointer.

func (*DiskBlobStore) Delete

func (d *DiskBlobStore) Delete(hash string) error

Delete deletes the blob from the store

func (*DiskBlobStore) Get

func (d *DiskBlobStore) Get(hash string) (stream.Blob, error)

Get returns the blob or an error if the blob doesn't exist.

func (*DiskBlobStore) Has

func (d *DiskBlobStore) Has(hash string) (bool, error)

Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.

func (*DiskBlobStore) Put

func (d *DiskBlobStore) Put(hash string, blob stream.Blob) error

Put stores the blob on disk

func (*DiskBlobStore) PutSD

func (d *DiskBlobStore) PutSD(hash string, blob stream.Blob) error

PutSD stores the sd blob on the disk

func (*DiskBlobStore) WipeOldestBlobs

func (d *DiskBlobStore) WipeOldestBlobs() (err error)

type MemoryBlobStore

type MemoryBlobStore struct {
	// contains filtered or unexported fields
}

MemoryBlobStore is an in memory only blob store with no persistence.

func NewMemoryBlobStore

func NewMemoryBlobStore() *MemoryBlobStore

func (*MemoryBlobStore) Debug

func (m *MemoryBlobStore) Debug() map[string]stream.Blob

Debug returns the blobs in memory. It's useful for testing and debugging.

func (*MemoryBlobStore) Delete

func (m *MemoryBlobStore) Delete(hash string) error

Delete deletes the blob from the store

func (*MemoryBlobStore) Get

func (m *MemoryBlobStore) Get(hash string) (stream.Blob, error)

Get returns the blob byte slice if present and errors if the blob is not found.

func (*MemoryBlobStore) Has

func (m *MemoryBlobStore) Has(hash string) (bool, error)

Has returns T/F if the blob is currently stored. It will never error.

func (*MemoryBlobStore) Put

func (m *MemoryBlobStore) Put(hash string, blob stream.Blob) error

Put stores the blob in memory

func (*MemoryBlobStore) PutSD

func (m *MemoryBlobStore) PutSD(hash string, blob stream.Blob) error

PutSD stores the sd blob in memory

type S3BlobStore

type S3BlobStore struct {
	// contains filtered or unexported fields
}

S3BlobStore is an S3 store

func NewS3BlobStore

func NewS3BlobStore(awsID, awsSecret, region, bucket string) *S3BlobStore

NewS3BlobStore returns an initialized S3 store pointer.

func (*S3BlobStore) Delete

func (s *S3BlobStore) Delete(hash string) error

func (*S3BlobStore) Get

func (s *S3BlobStore) Get(hash string) (stream.Blob, error)

Get returns the blob slice if present or errors on S3.

func (*S3BlobStore) Has

func (s *S3BlobStore) Has(hash string) (bool, error)

Has returns T/F or Error ( from S3 ) if the store contains the blob.

func (*S3BlobStore) Put

func (s *S3BlobStore) Put(hash string, blob stream.Blob) error

Put stores the blob on S3 or errors if S3 connection errors.

func (*S3BlobStore) PutSD

func (s *S3BlobStore) PutSD(hash string, blob stream.Blob) error

PutSD stores the sd blob on S3 or errors if S3 connection errors.