storage

package
v2.2.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 23, 2023 License: MIT Imports: 22 Imported by: 1

Documentation

Index

Constants

View Source 
const LockFile = "store.lock"

LockFile is our standard lockfile name.

Variables

View Source 
var (
	// ErrClosed is returned on operations on a closed storage
	ErrClosed = new_error("closed")

	// ErrNotFound is the error returned when a key cannot be found in storage
	ErrNotFound = new_error("key not found")

	// ErrAlreadyExist is the error returned when a key already exists in storage
	ErrAlreadyExists = new_error("key already exists")

	// ErrInvalidkey is the error returned when an invalid key is passed to storage
	ErrInvalidKey = new_error("invalid key")

	// ErrAlreadyLocked is returned on fail opening a storage lockfile
	ErrAlreadyLocked = new_error("storage lock already open")
)
View Source 
var DefaultDiskConfig = &DiskConfig{
	Overwrite:    true,
	WriteBufSize: 4096,
	Transform:    NopTransform(),
	Compression:  NoCompression(),
}

DefaultDiskConfig is the default DiskStorage configuration.

View Source 
var DefaultS3Config = &S3Config{
	CoreOpts:     minio.Options{},
	GetOpts:      minio.GetObjectOptions{},
	PutOpts:      minio.PutObjectOptions{},
	PutChunkOpts: minio.PutObjectPartOptions{},
	PutChunkSize: 4 * 1024 * 1024,
	StatOpts:     minio.StatObjectOptions{},
	RemoveOpts:   minio.RemoveObjectOptions{},
	ListSize:     200,
}

DefaultS3Config is the default S3Storage configuration.

Functions

This section is empty.

Types

type Compressor 

type Compressor interface {
	// Reader returns a new decompressing io.ReadCloser based on supplied (compressed) io.Reader
	Reader(io.ReadCloser) (io.ReadCloser, error)

	// Writer returns a new compressing io.WriteCloser based on supplied (uncompressed) io.Writer
	Writer(io.WriteCloser) (io.WriteCloser, error)
}

Compressor defines a means of compressing/decompressing values going into a key-value store

func GZipCompressor 

func GZipCompressor() Compressor

GZipCompressor returns a new Compressor that implements GZip at default compression level

func GZipCompressorLevel 

func GZipCompressorLevel(level int) Compressor

GZipCompressorLevel returns a new Compressor that implements GZip at supplied compression level

func NoCompression 

func NoCompression() Compressor

NoCompression is a Compressor that simply does nothing.

func SnappyCompressor 

func SnappyCompressor() Compressor

SnappyCompressor returns a new Compressor that implements Snappy.

func ZLibCompressor 

func ZLibCompressor() Compressor

ZLibCompressor returns a new Compressor that implements ZLib at default compression level

func ZLibCompressorLevel 

func ZLibCompressorLevel(level int) Compressor

ZLibCompressorLevel returns a new Compressor that implements ZLib at supplied compression level

func ZLibCompressorLevelDict 

func ZLibCompressorLevelDict(level int, dict []byte) Compressor

ZLibCompressorLevelDict returns a new Compressor that implements ZLib at supplied compression level with supplied dict

type DiskConfig 

type DiskConfig struct {
	// Transform is the supplied key <--> path KeyTransform.
	Transform KeyTransform

	// WriteBufSize is the buffer size to use when writing file streams.
	WriteBufSize int

	// Overwrite allows overwriting values of stored keys in the storage.
	Overwrite bool

	// LockFile allows specifying the filesystem path to use for the lockfile,
	// providing only a filename it will store the lockfile within provided store
	// path and nest the store under `path/store` to prevent access to lockfile.
	LockFile string

	// Compression is the Compressor to use when reading / writing files,
	// default is no compression.
	Compression Compressor
}

DiskConfig defines options to be used when opening a DiskStorage.

type DiskStorage 

type DiskStorage struct {
	// contains filtered or unexported fields
}

DiskStorage is a Storage implementation that stores directly to a filesystem.

func OpenDisk 

func OpenDisk(path string, cfg *DiskConfig) (*DiskStorage, error)

OpenDisk opens a DiskStorage instance for given folder path and configuration.

func (*DiskStorage) Clean 

func (st *DiskStorage) Clean(ctx context.Context) error

Clean implements Storage.Clean().

func (*DiskStorage) Close 

func (st *DiskStorage) Close() error

Close implements Storage.Close().

func (*DiskStorage) Filepath added in v2.2.2 

func (st *DiskStorage) Filepath(key string) (string, error)

Filepath checks and returns a formatted Filepath for given key.

func (*DiskStorage) ReadBytes 

func (st *DiskStorage) ReadBytes(ctx context.Context, key string) ([]byte, error)

ReadBytes implements Storage.ReadBytes().

func (*DiskStorage) ReadStream 

func (st *DiskStorage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error)

ReadStream implements Storage.ReadStream().

func (*DiskStorage) Remove 

func (st *DiskStorage) Remove(ctx context.Context, key string) error

Remove implements Storage.Remove().

func (*DiskStorage) Stat 

func (st *DiskStorage) Stat(ctx context.Context, key string) (bool, error)

Stat implements Storage.Stat().

func (*DiskStorage) WalkKeys 

func (st *DiskStorage) WalkKeys(ctx context.Context, opts WalkKeysOptions) error

WalkKeys implements Storage.WalkKeys().

func (*DiskStorage) WriteBytes 

func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte) (int, error)

WriteBytes implements Storage.WriteBytes().

func (*DiskStorage) WriteStream 

func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error)

WriteStream implements Storage.WriteStream().

type Entry 

type Entry struct {
	// Key is this entry's unique storage key.
	Key string

	// Size is the size of this entry in storage.
	// Note that size < 0 indicates unknown.
	Size int64
}

Entry represents a key in a Storage{} implementation, with any associated metadata that may have been set.

type KeyTransform 

type KeyTransform interface {
	// KeyToPath converts a supplied key to storage path
	KeyToPath(string) string

	// PathToKey converts a supplied storage path to key
	PathToKey(string) string
}

KeyTransform defines a method of converting store keys to storage paths (and vice-versa)

func NopTransform 

func NopTransform() KeyTransform

NopTransform returns a nop key transform (i.e. key = path)

type Lock 

type Lock struct {
	// contains filtered or unexported fields
}

Lock represents a filesystem lock to ensure only one storage instance open per path.

func OpenLock 

func OpenLock(path string) (*Lock, error)

OpenLock opens a lockfile at path.

func (*Lock) Close 

func (f *Lock) Close() error

Close will attempt to close the lockfile and file descriptor.

func (*Lock) Closed 

func (f *Lock) Closed() bool

Closed will return whether this lockfile has been closed (and unlocked).

type MemoryStorage 

type MemoryStorage struct {
	// contains filtered or unexported fields
}

MemoryStorage is a storage implementation that simply stores key-value pairs in a Go map in-memory. The map is protected by a mutex.

func OpenMemory 

func OpenMemory(size int, overwrites bool) *MemoryStorage

OpenMemory opens a new MemoryStorage instance with internal map starting size.

func (*MemoryStorage) Clean 

func (st *MemoryStorage) Clean(ctx context.Context) error

Clean implements Storage.Clean().

func (*MemoryStorage) Close 

func (st *MemoryStorage) Close() error

Close implements Storage.Close().

func (*MemoryStorage) ReadBytes 

func (st *MemoryStorage) ReadBytes(ctx context.Context, key string) ([]byte, error)

ReadBytes implements Storage.ReadBytes().

func (*MemoryStorage) ReadStream 

func (st *MemoryStorage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error)

ReadStream implements Storage.ReadStream().

func (*MemoryStorage) Remove 

func (st *MemoryStorage) Remove(ctx context.Context, key string) error

Remove implements Storage.Remove().

func (*MemoryStorage) Stat 

func (st *MemoryStorage) Stat(ctx context.Context, key string) (bool, error)

Stat implements Storage.Stat().

func (*MemoryStorage) WalkKeys 

func (st *MemoryStorage) WalkKeys(ctx context.Context, opts WalkKeysOptions) error

WalkKeys implements Storage.WalkKeys().

func (*MemoryStorage) WriteBytes 

func (st *MemoryStorage) WriteBytes(ctx context.Context, key string, b []byte) (int, error)

WriteBytes implements Storage.WriteBytes().

func (*MemoryStorage) WriteStream 

func (st *MemoryStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error)

WriteStream implements Storage.WriteStream().

type S3Config 

type S3Config struct {
	// CoreOpts are S3 client options passed during initialization.
	CoreOpts minio.Options

	// GetOpts are S3 client options passed during .Read___() calls.
	GetOpts minio.GetObjectOptions

	// PutOpts are S3 client options passed during .Write___() calls.
	PutOpts minio.PutObjectOptions

	// PutChunkSize is the chunk size (in bytes) to use when sending
	// a byte stream reader of unknown size as a multi-part object.
	PutChunkSize int64

	// PutChunkOpts are S3 client options passed during chunked .Write___() calls.
	PutChunkOpts minio.PutObjectPartOptions

	// StatOpts are S3 client options passed during .Stat() calls.
	StatOpts minio.StatObjectOptions

	// RemoveOpts are S3 client options passed during .Remove() calls.
	RemoveOpts minio.RemoveObjectOptions

	// ListSize determines how many items to include in each
	// list request, made during calls to .WalkKeys().
	ListSize int
}

S3Config defines options to be used when opening an S3Storage, mostly options for underlying S3 client library.

type S3Storage 

type S3Storage struct {
	// contains filtered or unexported fields
}

S3Storage is a storage implementation that stores key-value pairs in an S3 instance at given endpoint with bucket name.

func OpenS3 

func OpenS3(endpoint string, bucket string, cfg *S3Config) (*S3Storage, error)

OpenS3 opens a new S3Storage instance with given S3 endpoint URL, bucket name and configuration.

func (*S3Storage) Clean 

func (st *S3Storage) Clean(ctx context.Context) error

Clean implements Storage.Clean().

func (*S3Storage) Client added in v2.0.2 

func (st *S3Storage) Client() *minio.Core

Client returns access to the underlying S3 client.

func (*S3Storage) Close 

func (st *S3Storage) Close() error

Close implements Storage.Close().

func (*S3Storage) ReadBytes 

func (st *S3Storage) ReadBytes(ctx context.Context, key string) ([]byte, error)

ReadBytes implements Storage.ReadBytes().

func (*S3Storage) ReadStream 

func (st *S3Storage) ReadStream(ctx context.Context, key string) (io.ReadCloser, error)

ReadStream implements Storage.ReadStream().

func (*S3Storage) Remove 

func (st *S3Storage) Remove(ctx context.Context, key string) error

Remove implements Storage.Remove().

func (*S3Storage) Stat 

func (st *S3Storage) Stat(ctx context.Context, key string) (bool, error)

Stat implements Storage.Stat().

func (*S3Storage) WalkKeys 

func (st *S3Storage) WalkKeys(ctx context.Context, opts WalkKeysOptions) error

WalkKeys implements Storage.WalkKeys().

func (*S3Storage) WriteBytes 

func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) (int, error)

WriteBytes implements Storage.WriteBytes().

func (*S3Storage) WriteStream 

func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error)

WriteStream implements Storage.WriteStream().

type Storage 

type Storage interface {
	// ReadBytes returns the byte value for key in storage
	ReadBytes(ctx context.Context, key string) ([]byte, error)

	// ReadStream returns an io.ReadCloser for the value bytes at key in the storage
	ReadStream(ctx context.Context, key string) (io.ReadCloser, error)

	// WriteBytes writes the supplied value bytes at key in the storage
	WriteBytes(ctx context.Context, key string, value []byte) (int, error)

	// WriteStream writes the bytes from supplied reader at key in the storage
	WriteStream(ctx context.Context, key string, r io.Reader) (int64, error)

	// Stat checks if the supplied key is in the storage
	Stat(ctx context.Context, key string) (bool, error)

	// Remove attempts to remove the supplied key-value pair from storage
	Remove(ctx context.Context, key string) error

	// Close will close the storage, releasing any file locks
	Close() error

	// Clean removes unused values and unclutters the storage (e.g. removing empty folders)
	Clean(ctx context.Context) error

	// WalkKeys walks the keys in the storage
	WalkKeys(ctx context.Context, opts WalkKeysOptions) error
}

Storage defines a means of storing and accessing key value pairs

type WalkKeysOptions 

type WalkKeysOptions struct {
	// WalkFn is the function to apply on each StorageEntry
	WalkFn func(context.Context, Entry) error
}

WalkKeysOptions defines how to walk the keys in a storage implementation

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.