crabfs

README

🦀 CrabFS

CrabFS is a distributed (p2p) blockstorage that is easily embeddable and supports file seeking without downloading the whole file.

See examples for a concrete example.

This is a big WIP. Use with caution and PR's are welcome

Documentation

Index

• Constants
• Variables
• func BasicFetcherNew(ctx context.Context, fs interfaces.Core, object *pb.CrabObject, ...) (interfaces.Fetcher, error)
• func BlockSlicerNew(reader io.Reader, blockSize int64, cipher cipher.Block) (interfaces.Slicer, error)
• func BucketCoreNew(fs interfaces.Core, privateKey crabfsCrypto.PrivKey, bucket string, ...) interfaces.Bucket
• func CacheKeyFromFilename(pk string, bucketFilename string) string
• func DHTNamespacePKValidatorNew() libp2pRecord.Validator
• func DHTNamespaceValidatorNew(ctx context.Context, pkResolver SwarmPublicKeyResolver) libp2pRecord.Validator
• func GarbageCollectorNew(ctx context.Context, interval time.Duration, ds ipfsDatastore.Batching, ...) (interfaces.GarbageCollector, error)
• func GenerateKeyPair() (crabfsCrypto.PrivKey, error)
• func GenerateKeyPairReader() (io.Reader, error)
• func HostNew(settings *options.Settings, ds ipfsDatastore.Batching, ...) (interfaces.Host, error)
• func KeyFromFilename(pk string, bucketFilename string) string
• func New(opts ...options.Option) (interfaces.Core, error)
• func ReadPrivateKey(in io.Reader) (crabfsCrypto.PrivKey, error)
• type BasicFetcher
  • func (fetcher *BasicFetcher) Close() error
  • func (fetcher *BasicFetcher) Context() context.Context
  • func (fetcher *BasicFetcher) Read(p []byte) (n int, err error)
  • func (fetcher *BasicFetcher) Seek(offset int64, whence int) (int64, error)
  • func (fetcher *BasicFetcher) Size() int64
• type BlockSlicer
  • func (slicer *BlockSlicer) Next() (*pb.BlockMetadata, blocks.Block, int64, error)
• type DHTNamespacePKValidatorV1
  • func (validator DHTNamespacePKValidatorV1) Select(key string, values [][]byte) (int, error)
  • func (validator DHTNamespacePKValidatorV1) Validate(key string, value []byte) error
• type DHTNamespaceValidatorV1
  • func (validator DHTNamespaceValidatorV1) Select(key string, values [][]byte) (int, error)
  • func (validator DHTNamespaceValidatorV1) Validate(key string, value []byte) error
• type Relay
  • func RelayNew(ctx context.Context, port uint, bootstrapPeers []string, id identity.Identity) (*Relay, error)
  • func (relay *Relay) Close() error
  • func (relay *Relay) GetAddrs() []string
  • func (relay *Relay) GetHostID() string
• type SwarmPublicKeyResolver

Constants

const (
	// ProtocolV1 crabfs version 1
	ProtocolV1 = "/crabfs/v1"
)

Variables

var (
	// ErrInvalidPrivateKey an invalid private key was supplied
	ErrInvalidPrivateKey = errors.New("Invalid private key")

	// ErrInvalidRoot an invalid path was given
	ErrInvalidRoot = errors.New("Invalid root path")

	// ErrInvalidBlockSize an invalid block size was given
	ErrInvalidBlockSize = errors.New("Invalid block size. Block size must be bigger than 0")

	// ErrInvalidOffset an invalid path was given
	ErrInvalidOffset = errors.New("Invalid offset")

	// ErrBlockNotFound the requested block was not found
	ErrBlockNotFound = errors.New("Block not found")

	// ErrObjectNotFound the requested object was not found
	ErrObjectNotFound = errors.New("Object not found")

	// ErrFileLocked the requested file is locked
	ErrFileLocked = errors.New("Request file is locked")

	// ErrFileLockedNotOwned the requested file is not locked by this peer
	ErrFileLockedNotOwned = errors.New("Request file is not locked by this peer")
)

Functions

func BasicFetcherNew

func BasicFetcherNew(ctx context.Context, fs interfaces.Core, object *pb.CrabObject, privateKey crabfsCrypto.PrivKey) (interfaces.Fetcher, error)

BasicFetcherNew creates a new basic fetcher

func BlockSlicerNew

func BlockSlicerNew(reader io.Reader, blockSize int64, cipher cipher.Block) (interfaces.Slicer, error)

BlockSlicerNew creates a new BlockSlicer with 'blockSize'

func BucketCoreNew

func BucketCoreNew(fs interfaces.Core, privateKey crabfsCrypto.PrivKey, bucket string, root string) interfaces.Bucket

BucketCoreNew creates a new bucket core io

func CacheKeyFromFilename

func CacheKeyFromFilename(pk string, bucketFilename string) string

CacheKeyFromFilename converts a file name to a cache key

func DHTNamespacePKValidatorNew

func DHTNamespacePKValidatorNew() libp2pRecord.Validator

DHTNamespacePKValidatorNew creates a new validator that validates for all versions

func DHTNamespaceValidatorNew

func DHTNamespaceValidatorNew(ctx context.Context, pkResolver SwarmPublicKeyResolver) libp2pRecord.Validator

DHTNamespaceValidatorNew creates a new validator that validates for all versions validates keys like: /crabfs/v1/<hash of filename>

func GarbageCollectorNew

func GarbageCollectorNew(ctx context.Context, interval time.Duration, ds ipfsDatastore.Batching, bs ipfsBlockstore.Blockstore) (interfaces.GarbageCollector, error)

GarbageCollectorNew basic implementation of blockstore garbage collector

func GenerateKeyPair

func GenerateKeyPair() (crabfsCrypto.PrivKey, error)

GenerateKeyPair generates a private and public keys ready to be used

func GenerateKeyPairReader

func GenerateKeyPairReader() (io.Reader, error)

GenerateKeyPairReader generates a private and public keys ready to be used

func HostNew

func HostNew(settings *options.Settings, ds ipfsDatastore.Batching, blockstore blockstore.Blockstore) (interfaces.Host, error)

HostNew creates a new host

func KeyFromFilename

func KeyFromFilename(pk string, bucketFilename string) string

KeyFromFilename converts a file name to a key

func New

func New(opts ...options.Option) (interfaces.Core, error)

New create a new CrabFS

func ReadPrivateKey

func ReadPrivateKey(in io.Reader) (crabfsCrypto.PrivKey, error)

ReadPrivateKey reads the key from a reader

Types

type BasicFetcher

type BasicFetcher struct {
	// contains filtered or unexported fields
}

BasicFetcher single peer block fetcher

func (*BasicFetcher) Close

func (fetcher *BasicFetcher) Close() error

func (*BasicFetcher) Context

func (fetcher *BasicFetcher) Context() context.Context

func (*BasicFetcher) Read

func (fetcher *BasicFetcher) Read(p []byte) (n int, err error)

func (*BasicFetcher) Seek

func (fetcher *BasicFetcher) Seek(offset int64, whence int) (int64, error)

func (*BasicFetcher) Size

func (fetcher *BasicFetcher) Size() int64

type BlockSlicer

type BlockSlicer struct {
	// contains filtered or unexported fields
}

BlockSlicer a simple block slicer

func (*BlockSlicer) Next

func (slicer *BlockSlicer) Next() (*pb.BlockMetadata, blocks.Block, int64, error)

type DHTNamespacePKValidatorV1

type DHTNamespacePKValidatorV1 struct {
}

DHTNamespacePKValidatorV1 validates the /crabfs keys on the dht datastore

func (DHTNamespacePKValidatorV1) Select

func (validator DHTNamespacePKValidatorV1) Select(key string, values [][]byte) (int, error)

Select selects the best record from the set of records (e.g., the newest).

Decisions made by select should be stable.

func (DHTNamespacePKValidatorV1) Validate

func (validator DHTNamespacePKValidatorV1) Validate(key string, value []byte) error

Validate validates the given record, returning an error if it's invalid (e.g., expired, signed by the wrong key, etc.).

type DHTNamespaceValidatorV1

type DHTNamespaceValidatorV1 struct {
	// contains filtered or unexported fields
}

DHTNamespaceValidatorV1 validates the /crabfs keys on the dht datastore

func (DHTNamespaceValidatorV1) Select

func (validator DHTNamespaceValidatorV1) Select(key string, values [][]byte) (int, error)

Select selects the best record from the set of records (e.g., the newest).

Decisions made by select should be stable.

func (DHTNamespaceValidatorV1) Validate

func (validator DHTNamespaceValidatorV1) Validate(key string, value []byte) error

Validate validates the given record, returning an error if it's invalid (e.g., expired, signed by the wrong key, etc.).

type Relay

type Relay struct {
	// contains filtered or unexported fields
}

Relay controls a relay server

func RelayNew

func RelayNew(ctx context.Context, port uint, bootstrapPeers []string, id identity.Identity) (*Relay, error)

RelayNew creates a new relay instance

func (*Relay) Close

func (relay *Relay) Close() error

Close closes this relay instance

func (*Relay) GetAddrs

func (relay *Relay) GetAddrs() []string

GetAddrs get the addresses that this node is bound to

func (*Relay) GetHostID

func (relay *Relay) GetHostID() string

GetHostID returns the id of this p2p relay

type SwarmPublicKeyResolver

type SwarmPublicKeyResolver func(ctx context.Context, hash string) (crabfsCrypto.PubKey, error)

SwarmPublicKeyResolver resolver type to be used with DHTNamespacePKValidator