flatfs

README

flatfs

flatfs is a simple datastore implementation inspired by gits on-disk object storage.

Documentation

Overview

Package flatfs is a Datastore implementation that stores all objects in a two-level directory structure in the local file system, regardless of the hierarchy of the keys.

Package flatfs is a Datastore implementation that stores all objects in a two-level directory structure in the local file system, regardless of the hierarchy of the keys.

Index

• Constants
• Variables
• func Create(path string, fun *ShardIdV1) error
• func DirIsEmpty(name string) (bool, error)
• func DowngradeV1toV0(path string) error
• func Move(oldPath string, newPath string, out io.Writer) error
• func UpgradeV0toV1(path string, prefixLen int) error
• func WriteReadme(dir string, id *ShardIdV1) error
• func WriteShardFunc(dir string, id *ShardIdV1) error
• type Datastore
  • func CreateOrOpen(path string, fun *ShardIdV1, sync bool) (*Datastore, error)
  • func Open(path string, sync bool) (*Datastore, error)
  • func (fs *Datastore) Batch() (datastore.Batch, error)
  • func (fs *Datastore) Close() error
  • func (fs *Datastore) Delete(key datastore.Key) error
  • func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error)
  • func (fs *Datastore) Has(key datastore.Key) (exists bool, err error)
  • func (*Datastore) IsThreadSafe()
  • func (fs *Datastore) Put(key datastore.Key, value interface{}) error
  • func (fs *Datastore) Query(q query.Query) (query.Results, error)
  • func (fs *Datastore) ShardStr() string
• type ShardFunc
• type ShardIdV1
  • func NextToLast(suffixLen int) *ShardIdV1
  • func ParseShardFunc(str string) (*ShardIdV1, error)
  • func Prefix(prefixLen int) *ShardIdV1
  • func ReadShardFunc(dir string) (*ShardIdV1, error)
  • func Suffix(suffixLen int) *ShardIdV1
  • func (f *ShardIdV1) Func() ShardFunc
  • func (f *ShardIdV1) String() string

Constants

const PREFIX = "/repo/flatfs/shard/"

const README_FN = "_README"

const SHARDING_FN = "SHARDING"

const SyncThreadsMax = 16

don't block more than 16 threads on sync opearation 16 should be able to sataurate most RAIDs in case of two used disks per write (RAID 1, 5) and queue depth of 2, 16 concurrent Sync calls should be able to saturate 16 HDDs RAID TODO: benchmark it out, maybe provide tweak parmeter

Variables

var (
	ErrDatastoreExists       = errors.New("datastore already exist")
	ErrDatastoreDoesNotExist = errors.New("datastore directory does not exist")
	ErrShardingFileMissing   = fmt.Errorf("%s file not found in datastore", SHARDING_FN)
)

var IPFS_DEF_SHARD = NextToLast(2)

var IPFS_DEF_SHARD_STR = IPFS_DEF_SHARD.String()

var README_IPFS_DEF_SHARD = `` /* 1121-byte string literal not displayed */

Functions

func Create

func Create(path string, fun *ShardIdV1) error

func DirIsEmpty

func DirIsEmpty(name string) (bool, error)

From: http://stackoverflow.com/questions/30697324/how-to-check-if-directory-on-path-is-empty

func DowngradeV1toV0

func DowngradeV1toV0(path string) error

func Move

func Move(oldPath string, newPath string, out io.Writer) error

func UpgradeV0toV1

func UpgradeV0toV1(path string, prefixLen int) error

func WriteReadme

func WriteReadme(dir string, id *ShardIdV1) error

func WriteShardFunc

func WriteShardFunc(dir string, id *ShardIdV1) error

Types

type Datastore

type Datastore struct {
	// contains filtered or unexported fields
}

func CreateOrOpen

func CreateOrOpen(path string, fun *ShardIdV1, sync bool) (*Datastore, error)

convenience method

func Open

func Open(path string, sync bool) (*Datastore, error)

func (*Datastore) Batch

func (fs *Datastore) Batch() (datastore.Batch, error)

func (*Datastore) Close

func (fs *Datastore) Close() error

func (*Datastore) Delete

func (fs *Datastore) Delete(key datastore.Key) error

func (*Datastore) Get

func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error)

func (*Datastore) Has

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error)

func (*Datastore) IsThreadSafe

func (*Datastore) IsThreadSafe()

func (*Datastore) Put

func (fs *Datastore) Put(key datastore.Key, value interface{}) error

func (*Datastore) Query

func (fs *Datastore) Query(q query.Query) (query.Results, error)

func (*Datastore) ShardStr

func (fs *Datastore) ShardStr() string

type ShardFunc

type ShardFunc func(string) string

type ShardIdV1

type ShardIdV1 struct {
	// contains filtered or unexported fields
}

func NextToLast

func NextToLast(suffixLen int) *ShardIdV1

func ParseShardFunc

func ParseShardFunc(str string) (*ShardIdV1, error)

func Prefix

func Prefix(prefixLen int) *ShardIdV1

func ReadShardFunc

func ReadShardFunc(dir string) (*ShardIdV1, error)

func Suffix

func Suffix(suffixLen int) *ShardIdV1

func (*ShardIdV1) Func

func (f *ShardIdV1) Func() ShardFunc

func (*ShardIdV1) String

func (f *ShardIdV1) String() string