Documentation ¶
Index ¶
- Variables
- type BlockConfig
- type BlockStorage
- func (st *BlockStorage) Clean() error
- func (st *BlockStorage) ReadBytes(key string) ([]byte, error)
- func (st *BlockStorage) ReadStream(key string) (io.ReadCloser, error)
- func (st *BlockStorage) Remove(key string) error
- func (st *BlockStorage) Stat(key string) (bool, error)
- func (st *BlockStorage) WalkKeys(opts *WalkKeysOptions) error
- func (st *BlockStorage) WriteBytes(key string, value []byte) error
- func (st *BlockStorage) WriteStream(key string, r io.Reader) error
- type Compressor
- type DiskConfig
- type DiskStorage
- func (st *DiskStorage) Clean() error
- func (st *DiskStorage) ReadBytes(key string) ([]byte, error)
- func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error)
- func (st *DiskStorage) Remove(key string) error
- func (st *DiskStorage) Stat(key string) (bool, error)
- func (st *DiskStorage) WalkKeys(opts *WalkKeysOptions) error
- func (st *DiskStorage) WriteBytes(key string, value []byte) error
- func (st *DiskStorage) WriteStream(key string, r io.Reader) error
- type KeyTransform
- type Storage
- type StorageEntry
- type WalkKeysOptions
Constants ¶
This section is empty.
Variables ¶
var ( // ErrNotFound is the error returned when a key cannot be found in storage ErrNotFound = errorString("store/storage: key not found") // ErrAlreadyExist is the error returned when a key already exists in storage ErrAlreadyExists = errorString("store/storage: key already exists") // ErrInvalidkey is the error returned when an invalid key is passed to storage ErrInvalidKey = errorString("store/storage: invalid key") )
var DefaultBlockConfig = &BlockConfig{ BlockSize: 1024 * 16, WriteBufSize: 4096, Overwrite: false, Compression: NoCompression(), }
DefaultBlockConfig is the default BlockStorage configuration
var DefaultDiskConfig = &DiskConfig{ Overwrite: true, WriteBufSize: 4096, Transform: NopTransform(), Compression: NoCompression(), }
DefaultDiskConfig is the default DiskStorage configuration
Functions ¶
This section is empty.
Types ¶
type BlockConfig ¶
type BlockConfig struct { // BlockSize is the chunking size to use when splitting and storing blocks of data BlockSize int // WriteBufSize is the buffer size to use when writing file streams (PutStream) WriteBufSize int // Overwrite allows overwriting values of stored keys in the storage Overwrite bool // Compression is the Compressor to use when reading / writing files, default is no compression Compression Compressor }
BlockConfig defines options to be used when opening a BlockStorage
type BlockStorage ¶
type BlockStorage struct {
// contains filtered or unexported fields
}
BlockStorage is a Storage implementation that stores input data as chunks on a filesystem. Each value is chunked into blocks of configured size and these blocks are stored with name equal to their base64-encoded SHA256 hash-sum. A "node" file is finally created containing an array of hashes contained within this value
func OpenBlock ¶
func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error)
OpenBlock opens a BlockStorage instance for given folder path and configuration
func (*BlockStorage) ReadBytes ¶
func (st *BlockStorage) ReadBytes(key string) ([]byte, error)
ReadBytes implements Storage.ReadBytes()
func (*BlockStorage) ReadStream ¶
func (st *BlockStorage) ReadStream(key string) (io.ReadCloser, error)
ReadStream implements Storage.ReadStream()
func (*BlockStorage) Remove ¶
func (st *BlockStorage) Remove(key string) error
Remove implements Storage.Remove()
func (*BlockStorage) Stat ¶
func (st *BlockStorage) Stat(key string) (bool, error)
Stat implements Storage.Stat()
func (*BlockStorage) WalkKeys ¶
func (st *BlockStorage) WalkKeys(opts *WalkKeysOptions) error
WalkKeys implements Storage.WalkKeys()
func (*BlockStorage) WriteBytes ¶
func (st *BlockStorage) WriteBytes(key string, value []byte) error
WriteBytes implements Storage.WriteBytes()
func (*BlockStorage) WriteStream ¶
func (st *BlockStorage) WriteStream(key string, r io.Reader) error
WriteStream implements Storage.WriteStream()
type Compressor ¶
type Compressor interface { // Reader returns a new decompressing io.ReadCloser based on supplied (compressed) io.Reader Reader(io.Reader) (io.ReadCloser, error) // Writer returns a new compressing io.WriteCloser based on supplied (uncompressed) io.Writer Writer(io.Writer) (io.WriteCloser, error) }
Compressor defines a means of compressing/decompressing values going into a key-value store
func GZipCompressor ¶
func GZipCompressor() Compressor
GZipCompressor returns a new Compressor that implements GZip at default compression level
func GZipCompressorLevel ¶
func GZipCompressorLevel(level int) Compressor
GZipCompressorLevel returns a new Compressor that implements GZip at supplied compression level
func NoCompression ¶
func NoCompression() Compressor
NoCompression is a Compressor that simply does nothing
func SnappyCompressor ¶
func SnappyCompressor() Compressor
SnappyCompressor returns a new Compressor that implements Snappy
func ZLibCompressor ¶
func ZLibCompressor() Compressor
ZLibCompressor returns a new Compressor that implements ZLib at default compression level
func ZLibCompressorLevel ¶
func ZLibCompressorLevel(level int) Compressor
ZLibCompressorLevel returns a new Compressor that implements ZLib at supplied compression level
func ZLibCompressorLevelDict ¶
func ZLibCompressorLevelDict(level int, dict []byte) Compressor
ZLibCompressorLevelDict returns a new Compressor that implements ZLib at supplied compression level with supplied dict
type DiskConfig ¶
type DiskConfig struct { // Transform is the supplied key<-->path KeyTransform Transform KeyTransform // WriteBufSize is the buffer size to use when writing file streams (PutStream) WriteBufSize int // Overwrite allows overwriting values of stored keys in the storage Overwrite bool // Compression is the Compressor to use when reading / writing files, default is no compression Compression Compressor }
DiskConfig defines options to be used when opening a DiskStorage
type DiskStorage ¶
type DiskStorage struct {
// contains filtered or unexported fields
}
DiskStorage is a Storage implementation that stores directly to a filesystem
func OpenFile ¶
func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error)
OpenFile opens a DiskStorage instance for given folder path and configuration
func (*DiskStorage) ReadBytes ¶
func (st *DiskStorage) ReadBytes(key string) ([]byte, error)
ReadBytes implements Storage.ReadBytes()
func (*DiskStorage) ReadStream ¶
func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error)
ReadStream implements Storage.ReadStream()
func (*DiskStorage) Remove ¶
func (st *DiskStorage) Remove(key string) error
Remove implements Storage.Remove()
func (*DiskStorage) Stat ¶
func (st *DiskStorage) Stat(key string) (bool, error)
Stat implements Storage.Stat()
func (*DiskStorage) WalkKeys ¶
func (st *DiskStorage) WalkKeys(opts *WalkKeysOptions) error
WalkKeys implements Storage.WalkKeys()
func (*DiskStorage) WriteBytes ¶
func (st *DiskStorage) WriteBytes(key string, value []byte) error
WriteBytes implements Storage.WriteBytes()
func (*DiskStorage) WriteStream ¶
func (st *DiskStorage) WriteStream(key string, r io.Reader) error
WriteStream implements Storage.WriteStream()
type KeyTransform ¶
type KeyTransform interface { // KeyToPath converts a supplied key to storage path KeyToPath(string) string // PathToKey converts a supplied storage path to key PathToKey(string) string }
KeyTransform defines a method of converting store keys to storage paths (and vice-versa)
func NopTransform ¶
func NopTransform() KeyTransform
NopTransform returns a nop key transform (i.e. key = path)
type Storage ¶
type Storage interface { // Clean removes unused values and unclutters the storage (e.g. removing empty folders) Clean() error // ReadBytes returns the byte value for key in storage ReadBytes(key string) ([]byte, error) // ReadStream returns an io.ReadCloser for the value bytes at key in the storage ReadStream(key string) (io.ReadCloser, error) // WriteBytes writes the supplied value bytes at key in the storage WriteBytes(key string, value []byte) error // WriteStream writes the bytes from supplied reader at key in the storage WriteStream(key string, r io.Reader) error // Stat checks if the supplied key is in the storage Stat(key string) (bool, error) // Remove attempts to remove the supplied key-value pair from storage Remove(key string) error // WalkKeys walks the keys in the storage WalkKeys(opts *WalkKeysOptions) error }
Storage defines a means of storing and accessing key value pairs
type StorageEntry ¶
type StorageEntry interface { // Key returns the storage entry's key Key() string }
StorageEntry defines a key in Storage
type WalkKeysOptions ¶
type WalkKeysOptions struct { // WalkFn is the function to apply on each StorageEntry WalkFn func(StorageEntry) }
WalkKeysOptions defines how to walk the keys in a storage implementation