donut

package
v0.0.0-...-f4bd7b1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 17, 2015 License: Apache-2.0, Apache-2.0 Imports: 36 Imported by: 0

README

Donut

donut - Donut (do not delete) on disk format implementation released under Apache license v2.

Documentation

Index

Constants

View Source 
const (
	BucketPrivate         = BucketACL("private")
	BucketPublicRead      = BucketACL("public-read")
	BucketPublicReadWrite = BucketACL("public-read-write")
)

different types of ACL's currently supported for buckets

Variables

This section is empty.

Functions

func CleanupWritersOnError 

func CleanupWritersOnError(writers []io.WriteCloser)

CleanupWritersOnError purge writers on error

func Delimiter 

func Delimiter(object, delimiter string) string

Delimiter delims the string at delimiter

func HasDelimiter 

func HasDelimiter(objects []string, delim string) []string

HasDelimiter provides a new slice from an input slice which has elements with a delimiter

func HasNoDelimiter 

func HasNoDelimiter(objects []string, delim string) []string

HasNoDelimiter provides a new slice from an input slice which has elements without delimiter

func IsValidBucket 

func IsValidBucket(bucket string) bool

IsValidBucket - verify bucket name in accordance with

func IsValidBucketACL 

func IsValidBucketACL(acl string) bool

IsValidBucketACL - is provided acl string supported

func IsValidDonut 

func IsValidDonut(donutName string) bool

IsValidDonut - verify donut name is correct

func IsValidObjectName 

func IsValidObjectName(object string) bool

IsValidObjectName - verify object name in accordance with

func IsValidPrefix 

func IsValidPrefix(prefix string) bool

IsValidPrefix - verify prefix name is correct, an empty prefix is valid

func RemoveDuplicates 

func RemoveDuplicates(slice []string) []string

RemoveDuplicates removes duplicate elements from a slice

func SaveConfig 

func SaveConfig(a *Config) *probe.Error

SaveConfig save donut config

func SetDonutConfigPath 

func SetDonutConfigPath(configPath string)

SetDonutConfigPath - set custom donut config path

func SortUnique 

func SortUnique(objects []string) []string

SortUnique sort a slice in lexical order, removing duplicate elements

func SplitDelimiter 

func SplitDelimiter(objects []string, delim string) []string

SplitDelimiter provides a new slice from an input slice by splitting a delimiter

func TrimPrefix 

func TrimPrefix(objects []string, prefix string) []string

TrimPrefix trims off a prefix string from all the elements in a given slice

Types

type API 

type API struct {
	// contains filtered or unexported fields
}

API - local variables

func (API) AbortMultipartUpload 

func (donut API) AbortMultipartUpload(bucket, key, uploadID string, signature *Signature) *probe.Error

AbortMultipartUpload - abort an incomplete multipart session

func (API) AttachNode 

func (donut API) AttachNode(hostname string, disks []string) *probe.Error

AttachNode - attach node

func (API) CompleteMultipartUpload 

func (donut API) CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error)

CompleteMultipartUpload - complete a multipart upload and persist the data

func (API) CreateObject 

func (donut API) CreateObject(bucket, key, expectedMD5Sum string, size int64, data io.Reader, metadata map[string]string, signature *Signature) (ObjectMetadata, *probe.Error)

CreateObject - create an object

func (API) CreateObjectPart 

func (donut API) CreateObjectPart(bucket, key, uploadID string, partID int, contentType, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (string, *probe.Error)

CreateObjectPart - create a part in a multipart session

func (API) DetachNode 

func (donut API) DetachNode(hostname string) *probe.Error

DetachNode - detach node

func (API) GetBucketMetadata 

func (donut API) GetBucketMetadata(bucket string, signature *Signature) (BucketMetadata, *probe.Error)

GetBucketMetadata -

func (API) GetObject 

func (donut API) GetObject(w io.Writer, bucket string, object string, start, length int64) (int64, *probe.Error)

GetObject - GET object from cache buffer

func (API) GetObjectMetadata 

func (donut API) GetObjectMetadata(bucket, key string, signature *Signature) (ObjectMetadata, *probe.Error)

GetObjectMetadata - get object metadata from cache

func (API) Heal 

func (donut API) Heal() *probe.Error

Heal - heal your donuts

func (API) Info 

func (donut API) Info() (nodeDiskMap map[string][]string, err *probe.Error)

Info - return info about donut configuration

func (API) ListBuckets 

func (donut API) ListBuckets(signature *Signature) ([]BucketMetadata, *probe.Error)

ListBuckets - List buckets from cache

func (API) ListMultipartUploads 

func (donut API) ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata, signature *Signature) (BucketMultipartResourcesMetadata, *probe.Error)

ListMultipartUploads - list incomplete multipart sessions for a given bucket

func (API) ListObjectParts 

func (donut API) ListObjectParts(bucket, key string, resources ObjectResourcesMetadata, signature *Signature) (ObjectResourcesMetadata, *probe.Error)

ListObjectParts - list parts from incomplete multipart session for a given object

func (API) ListObjects 

func (donut API) ListObjects(bucket string, resources BucketResourcesMetadata, signature *Signature) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error)

ListObjects - list objects from cache

func (API) MakeBucket 

func (donut API) MakeBucket(bucketName, acl string, location io.Reader, signature *Signature) *probe.Error

MakeBucket - create bucket in cache

func (API) NewMultipartUpload 

func (donut API) NewMultipartUpload(bucket, key, contentType string, signature *Signature) (string, *probe.Error)

NewMultipartUpload - initiate a new multipart session

func (API) Rebalance 

func (donut API) Rebalance() *probe.Error

Rebalance - rebalance an existing donut with new disks and nodes

func (API) SetBucketMetadata 

func (donut API) SetBucketMetadata(bucket string, metadata map[string]string, signature *Signature) *probe.Error

SetBucketMetadata -

type APINotImplemented 

type APINotImplemented struct {
	API string
}

APINotImplemented - generic API not implemented error

func (APINotImplemented) Error 

func (e APINotImplemented) Error() string

Return string an error formatted as the given text

type AllBuckets 

type AllBuckets struct {
	Version string                    `json:"version"`
	Buckets map[string]BucketMetadata `json:"buckets"`
}

AllBuckets container for all buckets

type BackendCorrupted 

type BackendCorrupted BackendError

BackendCorrupted - path has corrupted data

func (BackendCorrupted) Error 

func (e BackendCorrupted) Error() string

Return string an error formatted as the given text

type BackendError 

type BackendError struct {
	Path string
}

BackendError - generic disk backend error

type BadDigest 

type BadDigest struct{}

BadDigest bad md5sum

func (BadDigest) Error 

func (e BadDigest) Error() string

type BucketACL 

type BucketACL string

BucketACL - bucket level access control

func (BucketACL) IsPrivate 

func (b BucketACL) IsPrivate() bool

IsPrivate - is acl Private

func (BucketACL) IsPublicRead 

func (b BucketACL) IsPublicRead() bool

IsPublicRead - is acl PublicRead

func (BucketACL) IsPublicReadWrite 

func (b BucketACL) IsPublicReadWrite() bool

IsPublicReadWrite - is acl PublicReadWrite

func (BucketACL) String 

func (b BucketACL) String() string

type BucketExists 

type BucketExists struct {
	Bucket string
}

BucketExists bucket exists

func (BucketExists) Error 

func (e BucketExists) Error() string

type BucketMetadata 

type BucketMetadata struct {
	Version       string                      `json:"version"`
	Name          string                      `json:"name"`
	ACL           BucketACL                   `json:"acl"`
	Created       time.Time                   `json:"created"`
	Multiparts    map[string]MultiPartSession `json:"multiparts"`
	Metadata      map[string]string           `json:"metadata"`
	BucketObjects map[string]struct{}         `json:"objects"`
}

BucketMetadata container for bucket level metadata

type BucketMultipartResourcesMetadata 

type BucketMultipartResourcesMetadata struct {
	KeyMarker          string
	UploadIDMarker     string
	NextKeyMarker      string
	NextUploadIDMarker string
	EncodingType       string
	MaxUploads         int
	IsTruncated        bool
	Upload             []*UploadMetadata
	Prefix             string
	Delimiter          string
	CommonPrefixes     []string
}

BucketMultipartResourcesMetadata - various types of bucket resources for inprogress multipart uploads

type BucketNameInvalid 

type BucketNameInvalid GenericBucketError

BucketNameInvalid - bucketname provided is invalid

func (BucketNameInvalid) Error 

func (e BucketNameInvalid) Error() string

Return string an error formatted as the given text

type BucketNotFound 

type BucketNotFound struct {
	Bucket string
}

BucketNotFound bucket does not exist

func (BucketNotFound) Error 

func (e BucketNotFound) Error() string

type BucketResourcesMetadata 

type BucketResourcesMetadata struct {
	Prefix         string
	Marker         string
	NextMarker     string
	Maxkeys        int
	EncodingType   string
	Delimiter      string
	IsTruncated    bool
	CommonPrefixes []string
}

BucketResourcesMetadata - various types of bucket resources

type ChecksumMismatch 

type ChecksumMismatch struct{}

ChecksumMismatch checksum mismatch

func (ChecksumMismatch) Error 

func (e ChecksumMismatch) Error() string

type CloudStorage 

type CloudStorage interface {
	// Storage service operations
	GetBucketMetadata(bucket string, signature *Signature) (BucketMetadata, *probe.Error)
	SetBucketMetadata(bucket string, metadata map[string]string, signature *Signature) *probe.Error
	ListBuckets(signature *Signature) ([]BucketMetadata, *probe.Error)
	MakeBucket(bucket string, ACL string, location io.Reader, signature *Signature) *probe.Error

	// Bucket operations
	ListObjects(string, BucketResourcesMetadata, *Signature) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error)

	// Object operations
	GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error)
	GetObjectMetadata(bucket, object string, signature *Signature) (ObjectMetadata, *probe.Error)
	// bucket, object, expectedMD5Sum, size, reader, metadata, signature
	CreateObject(string, string, string, int64, io.Reader, map[string]string, *Signature) (ObjectMetadata, *probe.Error)

	Multipart
}

CloudStorage is a donut cloud storage interface

type CompleteMultipartUpload 

type CompleteMultipartUpload struct {
	Part []CompletePart
}

CompleteMultipartUpload container for completing multipart upload

type CompletePart 

type CompletePart struct {
	PartNumber int
	ETag       string
}

CompletePart - completed part container

type Config 

type Config struct {
	Version     string              `json:"version"`
	MaxSize     uint64              `json:"max-size"`
	DonutName   string              `json:"donut-name"`
	NodeDiskMap map[string][]string `json:"node-disk-map"`
}

Config donut config

func LoadConfig 

func LoadConfig() (*Config, *probe.Error)

LoadConfig load donut config

type CorruptedBackend 

type CorruptedBackend struct {
	Backend string
}

CorruptedBackend backend found to be corrupted

func (CorruptedBackend) Error 

func (e CorruptedBackend) Error() string

type DigestError 

type DigestError struct {
	Bucket string
	Key    string
	Md5    string
}

DigestError - Generic Md5 error

type EntityTooLarge 

type EntityTooLarge struct {
	GenericObjectError
	Size    string
	MaxSize string
}

EntityTooLarge - object size exceeds maximum limit

func (EntityTooLarge) Error 

func (e EntityTooLarge) Error() string

Return string an error formatted as the given text

type GenericBucketError 

type GenericBucketError struct {
	Bucket string
}

GenericBucketError - generic bucket error

type GenericObjectError 

type GenericObjectError struct {
	Bucket string
	Object string
}

GenericObjectError - generic object error

type ImplementationError 

type ImplementationError struct {
	Bucket string
	Object string
	Err    error
}

ImplementationError - generic implementation error

func EmbedError 

func EmbedError(bucket, object string, err error) ImplementationError

EmbedError - wrapper function for error object

func (ImplementationError) Error 

func (e ImplementationError) Error() string

Return string an error formatted as the given text

type IncompleteBody 

type IncompleteBody GenericObjectError

IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header

func (IncompleteBody) Error 

func (e IncompleteBody) Error() string

Return string an error formatted as the given text

type Interface 

type Interface interface {
	CloudStorage
	Management
}

Interface is a collection of cloud storage and management interface

func New 

func New() (Interface, *probe.Error)

New instantiate a new donut

type InternalError 

type InternalError struct {
}

InternalError - generic internal error

func (InternalError) Error 

func (e InternalError) Error() string

Return string an error formatted as the given text

type InvalidACL 

type InvalidACL struct {
	ACL string
}

InvalidACL - acl invalid

func (InvalidACL) Error 

func (e InvalidACL) Error() string

type InvalidArgument 

type InvalidArgument struct{}

InvalidArgument invalid argument

func (InvalidArgument) Error 

func (e InvalidArgument) Error() string

type InvalidDigest 

type InvalidDigest DigestError

InvalidDigest - md5 in request header invalid

func (InvalidDigest) Error 

func (e InvalidDigest) Error() string

Return string an error formatted as the given text

type InvalidDisksArgument 

type InvalidDisksArgument struct{}

InvalidDisksArgument invalid number of disks per node

func (InvalidDisksArgument) Error 

func (e InvalidDisksArgument) Error() string

type InvalidErasureTechnique 

type InvalidErasureTechnique struct {
	Technique string
}

InvalidErasureTechnique invalid erasure technique

func (InvalidErasureTechnique) Error 

func (e InvalidErasureTechnique) Error() string

type InvalidPart 

type InvalidPart struct{}

InvalidPart One or more of the specified parts could not be found

func (InvalidPart) Error 

func (e InvalidPart) Error() string

type InvalidPartOrder 

type InvalidPartOrder struct {
	UploadID string
}

InvalidPartOrder parts are not ordered as Requested

func (InvalidPartOrder) Error 

func (e InvalidPartOrder) Error() string

type InvalidRange 

type InvalidRange struct {
	Start  int64
	Length int64
}

InvalidRange - invalid range

func (InvalidRange) Error 

func (e InvalidRange) Error() string

type InvalidUploadID 

type InvalidUploadID struct {
	UploadID string
}

InvalidUploadID invalid upload id

func (InvalidUploadID) Error 

func (e InvalidUploadID) Error() string

type ListObjectsResults 

type ListObjectsResults struct {
	Objects        map[string]ObjectMetadata `json:"objects"`
	CommonPrefixes []string                  `json:"commonPrefixes"`
	IsTruncated    bool                      `json:"isTruncated"`
}

ListObjectsResults container for list objects response

type MalformedXML 

type MalformedXML struct{}

MalformedXML invalid xml format

func (MalformedXML) Error 

func (e MalformedXML) Error() string

type Management 

type Management interface {
	Heal() *probe.Error
	Rebalance() *probe.Error
	Info() (map[string][]string, *probe.Error)

	AttachNode(hostname string, disks []string) *probe.Error
	DetachNode(hostname string) *probe.Error
}

Management is a donut management system interface

type Metadata 

type Metadata struct {
	Version string `json:"version"`
}

Metadata container for donut metadata

type MissingDateHeader 

type MissingDateHeader struct{}

MissingDateHeader date header missing

func (MissingDateHeader) Error 

func (e MissingDateHeader) Error() string

type MissingErasureTechnique 

type MissingErasureTechnique struct{}

MissingErasureTechnique missing erasure technique

func (MissingErasureTechnique) Error 

func (e MissingErasureTechnique) Error() string

type MultiPartSession 

type MultiPartSession struct {
	UploadID   string                  `json:"uploadId"`
	Initiated  time.Time               `json:"initiated"`
	Parts      map[string]PartMetadata `json:"parts"`
	TotalParts int                     `json:"total-parts"`
}

MultiPartSession multipart session

type Multipart 

type Multipart interface {
	NewMultipartUpload(bucket, key, contentType string, signature *Signature) (string, *probe.Error)
	AbortMultipartUpload(bucket, key, uploadID string, signature *Signature) *probe.Error
	CreateObjectPart(string, string, string, int, string, string, int64, io.Reader, *Signature) (string, *probe.Error)
	CompleteMultipartUpload(bucket, key, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error)
	ListMultipartUploads(string, BucketMultipartResourcesMetadata, *Signature) (BucketMultipartResourcesMetadata, *probe.Error)
	ListObjectParts(string, string, ObjectResourcesMetadata, *Signature) (ObjectResourcesMetadata, *probe.Error)
}

Multipart API

type NotImplemented 

type NotImplemented struct {
	Function string
}

NotImplemented function not implemented

func (NotImplemented) Error 

func (e NotImplemented) Error() string

type ObjectCorrupted 

type ObjectCorrupted struct {
	Object string
}

ObjectCorrupted object found to be corrupted

func (ObjectCorrupted) Error 

func (e ObjectCorrupted) Error() string

type ObjectExists 

type ObjectExists struct {
	Object string
}

ObjectExists object exists

func (ObjectExists) Error 

func (e ObjectExists) Error() string

type ObjectMetadata 

type ObjectMetadata struct {
	// version
	Version string `json:"version"`

	// object metadata
	Created time.Time `json:"created"`
	Bucket  string    `json:"bucket"`
	Object  string    `json:"object"`
	Size    int64     `json:"size"`

	// erasure
	DataDisks        uint8  `json:"sys.erasureK"`
	ParityDisks      uint8  `json:"sys.erasureM"`
	ErasureTechnique string `json:"sys.erasureTechnique"`
	BlockSize        int    `json:"sys.blockSize"`
	ChunkCount       int    `json:"sys.chunkCount"`

	// checksums
	MD5Sum    string `json:"sys.md5sum"`
	SHA512Sum string `json:"sys.sha512sum"`

	// metadata
	Metadata map[string]string `json:"metadata"`
}

ObjectMetadata container for object on donut system

type ObjectNameInvalid 

type ObjectNameInvalid GenericObjectError

ObjectNameInvalid - object name provided is invalid

func (ObjectNameInvalid) Error 

func (e ObjectNameInvalid) Error() string

Return string an error formatted as the given text

type ObjectNotFound 

type ObjectNotFound struct {
	Object string
}

ObjectNotFound object does not exist

func (ObjectNotFound) Error 

func (e ObjectNotFound) Error() string

type ObjectResourcesMetadata 

type ObjectResourcesMetadata struct {
	Bucket               string
	EncodingType         string
	Key                  string
	UploadID             string
	StorageClass         string
	PartNumberMarker     int
	NextPartNumberMarker int
	MaxParts             int
	IsTruncated          bool

	Part []*PartMetadata
}

ObjectResourcesMetadata - various types of object resources

type OperationNotPermitted 

type OperationNotPermitted struct {
	Op     string
	Reason string
}

OperationNotPermitted - operation not permitted

func (OperationNotPermitted) Error 

func (e OperationNotPermitted) Error() string

type ParityOverflow 

type ParityOverflow struct{}

ParityOverflow parity over flow

func (ParityOverflow) Error 

func (e ParityOverflow) Error() string

type PartMetadata 

type PartMetadata struct {
	PartNumber   int
	LastModified time.Time
	ETag         string
	Size         int64
}

PartMetadata - various types of individual part resources

type ProxyWriter 

type ProxyWriter struct {
	// contains filtered or unexported fields
}

ProxyWriter implements io.Writer to trap written bytes

func NewProxyWriter 

func NewProxyWriter(w io.Writer) *ProxyWriter

NewProxyWriter - wrap around a given writer with ProxyWriter

func (*ProxyWriter) Write 

func (r *ProxyWriter) Write(p []byte) (n int, err error)

type Signature 

type Signature struct {
	AccessKeyID     string
	SecretAccessKey string
	AuthHeader      string
	Request         *http.Request
}

Signature - local variables

func (*Signature) DoesSignatureMatch 

func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error)

DoesSignatureMatch - Verify authorization header with calculated header in accordance with - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html returns true if matches, false other wise if error is not nil then it is always false

type SignatureDoesNotMatch 

type SignatureDoesNotMatch struct {
	SignatureSent       string
	SignatureCalculated string
}

SignatureDoesNotMatch invalid signature

func (SignatureDoesNotMatch) Error 

func (e SignatureDoesNotMatch) Error() string

type TooManyBuckets 

type TooManyBuckets GenericBucketError

TooManyBuckets - total buckets exceeded

func (TooManyBuckets) Error 

func (e TooManyBuckets) Error() string

Return string an error formatted as the given text

type UnsupportedFilesystem 

type UnsupportedFilesystem struct {
	Type string
}

UnsupportedFilesystem unsupported filesystem type

func (UnsupportedFilesystem) Error 

func (e UnsupportedFilesystem) Error() string

type UploadMetadata 

type UploadMetadata struct {
	Key          string
	UploadID     string
	StorageClass string
	Initiated    time.Time
}

UploadMetadata container capturing metadata on in progress multipart upload in a given bucket

Source Files

View all Source files

Directories

Path Synopsis
cache
data
Package data implements in memory caching methods for data
 Package data implements in memory caching methods for data
metadata
Package metadata implements in memory caching methods for metadata information
 Package metadata implements in memory caching methods for metadata information

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.