cafs

README

cafs

Content-Addressable File System.

This is the data caching back-end used by the BitWrk distributed computing software. See https://bitwrk.net/ for more info.

Stores data in de-duplicated form and provides a remote-synching mechanism with another CAFS instance.

Data no longer referenced is kept in cache until the space is needed. Currently, data is not saved to persistent storage.

Documentation

Overview

Package cafs specifies a content-addressable file storage with support for de-duplication and remote syncing.

Index

• Variables
• type BoundedStorage
• type File
• type FileIterator
• type FileStorage
• type Printer
  • func NewWriterPrinter(w io.Writer) Printer
• type SKey
  • func MustParseKey(s string) *SKey
  • func ParseKey(s string) (*SKey, error)
  • func (k SKey) MarshalJSON() ([]byte, error)
  • func (k SKey) String() string
  • func (k *SKey) UnmarshalJSON(b []byte) error
• type Temporary
• type UsageInfo
  • func (ui UsageInfo) String() string

Variables

var ErrInvalidState = errors.New("Invalid temporary state")

var ErrNotEnoughSpace = errors.New("Not enough space")

var ErrNotFound = errors.New("Not found")

var ErrStillOpen = errors.New("Temporary still open")

var LoggingEnabled = false

Types

type BoundedStorage

type BoundedStorage interface {
	FileStorage

	GetUsageInfo() UsageInfo

	// Clears any data that is not locked externally and returns the number of bytes freed.
	FreeCache() int64
}

Interface BoundedStorage describes file storage with bounded capacity

type File

type File interface {
	// Signals that this file handle is no longer in use.
	// If no handles exist on a file anymore, the storage space
	// bound by it can ce reclaimed by the garbage collector.
	// It is an error to call Open() or Duplicate() after Dispose().
	// It is ok to call Dispose() more than once.
	Dispose()
	Key() SKey
	Open() io.ReadCloser
	Size() int64
	// Creates a new handle to the same file that must be Dispose()'d
	// independently.
	Duplicate() File

	// Returns true if the file is stored in chunks internally.
	// It is an error to call this function after Dispose().
	IsChunked() bool
	// Returns an iterator to the chunks of the file. The iterator must be disposed after use.
	Chunks() FileIterator
	// Returns the number of chunks in this file, or 1 if file is not chunked
	NumChunks() int64
}

type FileIterator

type FileIterator interface {
	// Must be called after using this iterator.
	Dispose()
	// Returns a copy of this iterator that must be Dispose()'d independently.
	Duplicate() FileIterator

	// Advances the iterator and returns true if successful, or false if no further chunks
	// could be read.
	// Must be called before calling File().
	Next() bool

	// Returns the key of the last file or chunk successfully read by Next().
	// Before calling this function, Next() must have been called and returned true.
	Key() SKey

	// Returns the size of the last file or chunk successfully read by Next().
	// Before calling this function, Next() must have been called and returned true.
	Size() int64

	// Returns the last file or chunk successfully read by Next() as a file.
	// The received File must be Dispose()'d.
	// Before calling this function, Next() must have been called and returned true.
	File() File
}

Iterate over a set of files or chunks.

type FileStorage

type FileStorage interface {
	// Creates a new temporary that can be written into. The info string will stick
	// with the temporary and also with the file, should it be created, and serves only
	// informational purposes.
	Create(info string) Temporary

	// Queries a file from the storage that can be read from. If the file exists, a File
	// interface is returned that has been locked once and that must be released correctly.
	// If the file does not exist, then (nil, ErrNotFound) is returned.
	Get(key *SKey) (File, error)

	DumpStatistics(log Printer)
}

type Printer

type Printer interface {
	Printf(format string, v ...interface{})
}

Interface Printer is used by CAFS for debugging output

func NewWriterPrinter

func NewWriterPrinter(w io.Writer) Printer

type SKey

type SKey [32]byte

func MustParseKey

func MustParseKey(s string) *SKey

func ParseKey

func ParseKey(s string) (*SKey, error)

func (SKey) MarshalJSON

func (k SKey) MarshalJSON() ([]byte, error)

func (SKey) String

func (k SKey) String() string

func (*SKey) UnmarshalJSON

func (k *SKey) UnmarshalJSON(b []byte) error

type Temporary

type Temporary interface {
	// Stores the temporary file into the FileStorage, where it
	// can be retrieved by key - after Close() has been called.
	io.WriteCloser

	// Returns a handle to the stored file, once Close() has been
	// called and no error occurred. Otherwise, panics.
	File() File

	// Must be called when the temporary is no longer needed.
	// It's ok to call Dispose() more than once.
	Dispose()
}

type UsageInfo

type UsageInfo struct {
	Used     int64 // The number of bytes used by the storage
	Capacity int64 // The maximum number of bytes usable by the storage
	Locked   int64 // The number of bytes currently locked by the storage
}

Type UsageInfo contains information about how many bytes are used, locked and available by a BoundedStorage.

func (UsageInfo) String

func (ui UsageInfo) String() string