storage

package
v1.3.8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 8, 2022 License: MIT Imports: 22 Imported by: 1

Documentation

Index

Constants

View Source 
const LockFile = "store.lock"

LockFile is our standard lockfile name.

Variables

View Source 
var (
	// ErrClosed is returned on operations on a closed storage
	ErrClosed = errors.New("store/storage: closed")

	// ErrNotFound is the error returned when a key cannot be found in storage
	ErrNotFound = errors.New("store/storage: key not found")

	// ErrAlreadyExist is the error returned when a key already exists in storage
	ErrAlreadyExists = errors.New("store/storage: key already exists")

	// ErrInvalidkey is the error returned when an invalid key is passed to storage
	ErrInvalidKey = errors.New("store/storage: invalid key")

	// ErrAlreadyLocked is returned on fail opening a storage lockfile
	ErrAlreadyLocked = errors.New("store/storage: storage lock already open")
)
View Source 
var DefaultBlockConfig = &BlockConfig{
	BlockSize:    1024 * 16,
	WriteBufSize: 4096,
	Overwrite:    false,
	Compression:  NoCompression(),
}

DefaultBlockConfig is the default BlockStorage configuration

View Source 
var DefaultDiskConfig = &DiskConfig{
	Overwrite:    true,
	WriteBufSize: 4096,
	Transform:    NopTransform(),
	Compression:  NoCompression(),
}

DefaultDiskConfig is the default DiskStorage configuration

Functions

This section is empty.

Types

type BlockConfig 

type BlockConfig struct {
	// BlockSize is the chunking size to use when splitting and storing blocks of data
	BlockSize int

	// ReadBufSize is the buffer size to use when reading node files
	ReadBufSize int

	// WriteBufSize is the buffer size to use when writing file streams (PutStream)
	WriteBufSize int

	// Overwrite allows overwriting values of stored keys in the storage
	Overwrite bool

	// Compression is the Compressor to use when reading / writing files, default is no compression
	Compression Compressor
}

BlockConfig defines options to be used when opening a BlockStorage

type BlockStorage 

type BlockStorage struct {
	// contains filtered or unexported fields
}

BlockStorage is a Storage implementation that stores input data as chunks on a filesystem. Each value is chunked into blocks of configured size and these blocks are stored with name equal to their base64-encoded SHA256 hash-sum. A "node" file is finally created containing an array of hashes contained within this value

func OpenBlock 

func OpenBlock(path string, cfg *BlockConfig) (*BlockStorage, error)

OpenBlock opens a BlockStorage instance for given folder path and configuration

func (*BlockStorage) Clean 

func (st *BlockStorage) Clean() error

Clean implements storage.Clean()

func (*BlockStorage) Close added in v1.2.0 

func (st *BlockStorage) Close() error

Close implements Storage.Close()

func (*BlockStorage) ReadBytes 

func (st *BlockStorage) ReadBytes(key string) ([]byte, error)

ReadBytes implements Storage.ReadBytes()

func (*BlockStorage) ReadStream 

func (st *BlockStorage) ReadStream(key string) (io.ReadCloser, error)

ReadStream implements Storage.ReadStream()

func (*BlockStorage) Remove 

func (st *BlockStorage) Remove(key string) error

Remove implements Storage.Remove()

func (*BlockStorage) Stat 

func (st *BlockStorage) Stat(key string) (bool, error)

Stat implements Storage.Stat()

func (*BlockStorage) WalkKeys 

func (st *BlockStorage) WalkKeys(opts WalkKeysOptions) error

WalkKeys implements Storage.WalkKeys()

func (*BlockStorage) WriteBytes 

func (st *BlockStorage) WriteBytes(key string, value []byte) error

WriteBytes implements Storage.WriteBytes()

func (*BlockStorage) WriteStream 

func (st *BlockStorage) WriteStream(key string, r io.Reader) error

WriteStream implements Storage.WriteStream()

type Compressor 

type Compressor interface {
	// Reader returns a new decompressing io.ReadCloser based on supplied (compressed) io.Reader
	Reader(io.Reader) (io.ReadCloser, error)

	// Writer returns a new compressing io.WriteCloser based on supplied (uncompressed) io.Writer
	Writer(io.Writer) (io.WriteCloser, error)
}

Compressor defines a means of compressing/decompressing values going into a key-value store

func GZipCompressor 

func GZipCompressor() Compressor

GZipCompressor returns a new Compressor that implements GZip at default compression level

func GZipCompressorLevel 

func GZipCompressorLevel(level int) Compressor

GZipCompressorLevel returns a new Compressor that implements GZip at supplied compression level

func NoCompression 

func NoCompression() Compressor

NoCompression is a Compressor that simply does nothing

func SnappyCompressor 

func SnappyCompressor() Compressor

SnappyCompressor returns a new Compressor that implements Snappy

func ZLibCompressor 

func ZLibCompressor() Compressor

ZLibCompressor returns a new Compressor that implements ZLib at default compression level

func ZLibCompressorLevel 

func ZLibCompressorLevel(level int) Compressor

ZLibCompressorLevel returns a new Compressor that implements ZLib at supplied compression level

func ZLibCompressorLevelDict 

func ZLibCompressorLevelDict(level int, dict []byte) Compressor

ZLibCompressorLevelDict returns a new Compressor that implements ZLib at supplied compression level with supplied dict

type DiskConfig 

type DiskConfig struct {
	// Transform is the supplied key<-->path KeyTransform
	Transform KeyTransform

	// WriteBufSize is the buffer size to use when writing file streams (PutStream)
	WriteBufSize int

	// Overwrite allows overwriting values of stored keys in the storage
	Overwrite bool

	// LockFile allows specifying the filesystem path to use for the lockfile,
	// providing only a filename it will store the lockfile within provided store
	// path and nest the store under `path/store` to prevent access to lockfile
	LockFile string

	// Compression is the Compressor to use when reading / writing files, default is no compression
	Compression Compressor
}

DiskConfig defines options to be used when opening a DiskStorage

type DiskStorage 

type DiskStorage struct {
	// contains filtered or unexported fields
}

DiskStorage is a Storage implementation that stores directly to a filesystem

func OpenFile 

func OpenFile(path string, cfg *DiskConfig) (*DiskStorage, error)

OpenFile opens a DiskStorage instance for given folder path and configuration

func (*DiskStorage) Clean 

func (st *DiskStorage) Clean() error

Clean implements Storage.Clean()

func (*DiskStorage) Close added in v1.2.0 

func (st *DiskStorage) Close() error

Close implements Storage.Close()

func (*DiskStorage) ReadBytes 

func (st *DiskStorage) ReadBytes(key string) ([]byte, error)

ReadBytes implements Storage.ReadBytes()

func (*DiskStorage) ReadStream 

func (st *DiskStorage) ReadStream(key string) (io.ReadCloser, error)

ReadStream implements Storage.ReadStream()

func (*DiskStorage) Remove 

func (st *DiskStorage) Remove(key string) error

Remove implements Storage.Remove()

func (*DiskStorage) Stat 

func (st *DiskStorage) Stat(key string) (bool, error)

Stat implements Storage.Stat()

func (*DiskStorage) WalkKeys 

func (st *DiskStorage) WalkKeys(opts WalkKeysOptions) error

WalkKeys implements Storage.WalkKeys()

func (*DiskStorage) WriteBytes 

func (st *DiskStorage) WriteBytes(key string, value []byte) error

WriteBytes implements Storage.WriteBytes()

func (*DiskStorage) WriteStream 

func (st *DiskStorage) WriteStream(key string, r io.Reader) error

WriteStream implements Storage.WriteStream()

type KeyTransform 

type KeyTransform interface {
	// KeyToPath converts a supplied key to storage path
	KeyToPath(string) string

	// PathToKey converts a supplied storage path to key
	PathToKey(string) string
}

KeyTransform defines a method of converting store keys to storage paths (and vice-versa)

func NopTransform 

func NopTransform() KeyTransform

NopTransform returns a nop key transform (i.e. key = path)

type Lock added in v1.3.0 

type Lock struct {
	// contains filtered or unexported fields
}

Lock represents a filesystem lock to ensure only one storage instance open per path.

func OpenLock added in v1.2.0 

func OpenLock(path string) (*Lock, error)

OpenLock opens a lockfile at path.

func (*Lock) Add added in v1.3.0 

func (f *Lock) Add()

Add will add '1' to the underlying sync.WaitGroup.

func (*Lock) Close added in v1.3.0 

func (f *Lock) Close() error

Close will attempt to close the lockfile and file descriptor.

func (*Lock) Closed added in v1.3.0 

func (f *Lock) Closed() bool

Closed will return whether this lockfile has been closed (and unlocked).

func (*Lock) Done added in v1.3.0 

func (f *Lock) Done()

Done will decrememnt '1' from the underlying sync.WaitGroup.

type MemoryStorage added in v1.1.0 

type MemoryStorage struct {
	// contains filtered or unexported fields
}

MemoryStorage is a storage implementation that simply stores key-value pairs in a Go map in-memory. The map is protected by a mutex.

func OpenMemory added in v1.1.0 

func OpenMemory(size int, overwrites bool) *MemoryStorage

OpenMemory opens a new MemoryStorage instance with internal map of 'size'.

func (*MemoryStorage) Clean added in v1.1.0 

func (st *MemoryStorage) Clean() error

Clean implements Storage.Clean().

func (*MemoryStorage) Close added in v1.2.0 

func (st *MemoryStorage) Close() error

Close implements Storage.Close().

func (*MemoryStorage) ReadBytes added in v1.1.0 

func (st *MemoryStorage) ReadBytes(key string) ([]byte, error)

ReadBytes implements Storage.ReadBytes().

func (*MemoryStorage) ReadStream added in v1.1.0 

func (st *MemoryStorage) ReadStream(key string) (io.ReadCloser, error)

ReadStream implements Storage.ReadStream().

func (*MemoryStorage) Remove added in v1.1.0 

func (st *MemoryStorage) Remove(key string) error

Remove implements Storage.Remove().

func (*MemoryStorage) Stat added in v1.1.0 

func (st *MemoryStorage) Stat(key string) (bool, error)

Stat implements Storage.Stat().

func (*MemoryStorage) WalkKeys added in v1.1.0 

func (st *MemoryStorage) WalkKeys(opts WalkKeysOptions) error

WalkKeys implements Storage.WalkKeys().

func (*MemoryStorage) WriteBytes added in v1.1.0 

func (st *MemoryStorage) WriteBytes(key string, b []byte) error

WriteBytes implements Storage.WriteBytes().

func (*MemoryStorage) WriteStream added in v1.1.0 

func (st *MemoryStorage) WriteStream(key string, r io.Reader) error

WriteStream implements Storage.WriteStream().

type Storage 

type Storage interface {
	// ReadBytes returns the byte value for key in storage
	ReadBytes(key string) ([]byte, error)

	// ReadStream returns an io.ReadCloser for the value bytes at key in the storage
	ReadStream(key string) (io.ReadCloser, error)

	// WriteBytes writes the supplied value bytes at key in the storage
	WriteBytes(key string, value []byte) error

	// WriteStream writes the bytes from supplied reader at key in the storage
	WriteStream(key string, r io.Reader) error

	// Stat checks if the supplied key is in the storage
	Stat(key string) (bool, error)

	// Remove attempts to remove the supplied key-value pair from storage
	Remove(key string) error

	// Close will close the storage, releasing any file locks
	Close() error

	// Clean removes unused values and unclutters the storage (e.g. removing empty folders)
	Clean() error

	// WalkKeys walks the keys in the storage
	WalkKeys(opts WalkKeysOptions) error
}

Storage defines a means of storing and accessing key value pairs

type StorageEntry 

type StorageEntry interface {
	// Key returns the storage entry's key
	Key() string
}

StorageEntry defines a key in Storage

type WalkKeysOptions 

type WalkKeysOptions struct {
	// WalkFn is the function to apply on each StorageEntry
	WalkFn func(StorageEntry)
}

WalkKeysOptions defines how to walk the keys in a storage implementation

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.