blob

README

Blob

File System based blob store with encryption,metadata and integrity check support

Features

Reader and Writer

Blob provides io.Reader and io.Writer interfaces to read and write binary data

	w, err := blob.Writer()
	if err != nil {
		log.Fatal(err)
	}

	r, err := blob.Reader()
	if err != nil {
		log.Fatal(err)
	}

Metadata

Blob provides APIs to store and retrieve metadata of the blob

	if err := blob.Put("key", "value"); err != nil {
		log.Fatal(err)
	}

	if v, err := blob.Get("key"); err != nil || v != "value" {
		log.Fatal("value not found")
	}

	if err := blob.PutAll(map[string]string{
		"key-1": "value-1",
		"key-2": "value-2",
	}); err != nil {
		log.Fatal(err)
	}

	m, err :=  blob.GetAll() 
	if err != nil {
		log.Fatal(err)
	}

Encryption (dual key encryption)

Blob content along with metadata can be encrypted by providing a primary encryption key. For each blob a random secondary encryption (aes256 bit) key is created to encrypt the blob content. Secondary encryption key along with the metadata is encrypted with the primary cipher.

	key := make([]byte, 32)
	if _, err := crand.Read(key); err != nil {
		log.Fatal(err)
	}

	aead, err := chacha20poly1305.New(key)
	if err != nil {
		log.Fatal(err)
	}

	bucket, err := blob.NewBucket(path, aead)
	if err != nil {
		log.Fatal(err)
	}

Integrity

Blobs can be sealed and verified. Once sealed, a HMAC sum of the blob content is calculated and stored in the metadata. Upon verification, the sum is verified against the blob content.

	// seal the blob
	if err := blob.Seal(); err != nil {
		log.Fatal(err)
	}

	// verify the glob
	if err := blob.Verify(); err != nil {
		log.Fatal("blob integrity compromised. error: %v", err)
	}

Documentation

Index

• Constants
• Variables
• type Blob
• type Bucket
  • func NewBucket(path string, aead cipher.AEAD) (*Bucket, error)
  • func (b *Bucket) Child(id string) (*Bucket, error)
  • func (b *Bucket) ID() string
  • func (b *Bucket) ListBlobs(cb ListBlobsFn) error
  • func (b *Bucket) ListBuckets(cb ListBucketsFn) error
  • func (b *Bucket) New(id string) (Blob, error)
  • func (b *Bucket) Open(id string) (Blob, error)
  • func (b *Bucket) Path() string
• type ListBlobsFn
• type ListBucketsFn

Examples

• Bucket.ListBlobs
• Bucket.ListBuckets
• NewBucket
• NewBucket (Encrypted)

Constants

const (
	KeySignature  = "_signature"
	KeyEncryption = "_encryption"
)

Blob metadata keys

const DefaultFilePerm = 0700

DefaultFilePerm is the default file permissions of recoding dirs and files.

Variables

var (
	ErrNotFound    = errors.New("blob: not found")
	ErrSigNotFound = errors.New("blob: signature not found")
	ErrSigMisMatch = errors.New("blob: signature mismatch")
	ErrSealed      = errors.New("blob: sealed")
)

Errors

Types

type Blob

type Blob interface {
	// ID returns the id of the blob
	ID() string
	// Reader return a reader to read the blob
	Reader() (io.Reader, error)
	// Writer return a writer to write to the blob
	Writer() (io.Writer, error)
	// Put puts the key value pair in the blob metadata
	Put(k, v string) error
	// Get gets the value for the given key from the blob metadata
	Get(k string) (string, error)
	// PutAll puts the map in the blob metadata
	PutAll(map[string]string) error
	// GetAll gets all the values from blob metadata
	GetAll() (map[string]string, error)
	// Size of the blob. returns 0 if file doesn't exists
	Size() int64
	// Timestamp returns the last modified timestamp of the blob
	Timestamp() (time.Time, error)
	// Seal seals the blob by storing the hash of the content.
	// After the blob is sealed, the blob will become readonly
	Seal() error
	// Verify verifies the blob hash value
	Verify() error
	// Close closes the blob
	Close() error
}

Blob represents a blob which is available to read/write/get/put binary data

type Bucket

type Bucket struct {
	// contains filtered or unexported fields
}

Bucket represents a collection os blobs and sub buckets

func NewBucket

func NewBucket(path string, aead cipher.AEAD) (*Bucket, error)

NewBucket creates a new Bucket for the given path If the cipher is nil the the blobs will not be encrypted

Example

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/smitajit/blob"
)

func main() {
	path, clean, err := tempdir()
	if err != nil {
		log.Fatal(err)
	}
	defer clean()

	bucket, err := blob.NewBucket(path, nil)
	if err != nil {
		log.Fatal(err)
	}

	blob, err := bucket.New("demo-blob")
	if err != nil {
		log.Fatal(err)
	}

	if err := blob.Put("foo", "bar"); err != nil {
		log.Fatal(err)
	}

	w, err := blob.Writer()
	if err != nil {
		log.Fatal(err)
	}

	if _, err := fmt.Fprintf(w, "hello world"); err != nil {
		log.Fatal(err)
	}

	val, err := blob.Get("foo")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("value: ", val)

	r, err := blob.Reader()
	if err != nil {
		log.Fatal(err)
	}

	p, err := ioutil.ReadAll(r)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("content: ", string(p))
}

type cleanfn func() error

func tempdir() (string, cleanfn, error) {
	dir, err := os.MkdirTemp(os.TempDir(), "blob")
	if err != nil {
		return "", nil, err
	}
	return dir, func() error {
		return os.RemoveAll(dir)
	}, nil
}

Output:

value:  bar
content:  hello world

Example (Encrypted)

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	crand "crypto/rand"

	"github.com/smitajit/blob"
	"golang.org/x/crypto/chacha20poly1305"
)

func main() {
	path, clean, err := tempdir()
	if err != nil {
		log.Fatal(err)
	}
	defer clean()

	key := make([]byte, 32)
	if _, err := crand.Read(key); err != nil {
		log.Fatal(err)
	}

	aead, err := chacha20poly1305.New(key)
	if err != nil {
		log.Fatal(err)
	}

	bucket, err := blob.NewBucket(path, aead)
	if err != nil {
		log.Fatal(err)
	}

	blob, err := bucket.New("demo-blob")
	if err != nil {
		log.Fatal(err)
	}

	if err := blob.Put("foo", "bar"); err != nil {
		log.Fatal(err)
	}

	w, err := blob.Writer()
	if err != nil {
		log.Fatal(err)
	}

	if _, err := fmt.Fprintf(w, "hello world"); err != nil {
		log.Fatal(err)
	}

	val, err := blob.Get("foo")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("value: ", val)

	r, err := blob.Reader()
	if err != nil {
		log.Fatal(err)
	}

	p, err := ioutil.ReadAll(r)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("content: ", string(p))
}

type cleanfn func() error

func tempdir() (string, cleanfn, error) {
	dir, err := os.MkdirTemp(os.TempDir(), "blob")
	if err != nil {
		return "", nil, err
	}
	return dir, func() error {
		return os.RemoveAll(dir)
	}, nil
}

Output:

value:  bar
content:  hello world

func (*Bucket) Child

func (b *Bucket) Child(id string) (*Bucket, error)

Child creates a new or returns an existing sub bucket in the bucket

func (*Bucket) ID

func (b *Bucket) ID() string

ID returns the ID of the bucket

func (*Bucket) ListBlobs

func (b *Bucket) ListBlobs(cb ListBlobsFn) error

ListBlobs lists the blobs inside the bucket

Example

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/smitajit/blob"
)

func main() {
	path, clean, err := tempdir()
	if err != nil {
		log.Fatal(err)
	}
	// fmt.Println(path)
	defer clean()

	b, err := blob.NewBucket(path, nil)
	if err != nil {
		log.Fatal(err)
	}

	_, err = b.New("blob-1")
	if err != nil {
		log.Fatal(err)
	}

	_, err = b.New("blob-2")
	if err != nil {
		log.Fatal(err)
	}

	b.ListBlobs(func(b blob.Blob) error {
		fmt.Println(b.ID())
		return nil
	})
}

type cleanfn func() error

func tempdir() (string, cleanfn, error) {
	dir, err := os.MkdirTemp(os.TempDir(), "blob")
	if err != nil {
		return "", nil, err
	}
	return dir, func() error {
		return os.RemoveAll(dir)
	}, nil
}

Output:

blob-1
blob-2

func (*Bucket) ListBuckets

func (b *Bucket) ListBuckets(cb ListBucketsFn) error

ListBuckets lists the sub buckets of the bucket

Example

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/smitajit/blob"
)

func main() {
	path, clean, err := tempdir()
	if err != nil {
		log.Fatal(err)
	}
	// fmt.Println(path)
	defer clean()

	b, err := blob.NewBucket(path, nil)
	if err != nil {
		log.Fatal(err)
	}

	_, err = b.Child("sub-bucket-1")
	if err != nil {
		log.Fatal(err)
	}

	_, err = b.Child("sub-bucket-2")
	if err != nil {
		log.Fatal(err)
	}

	b.ListBuckets(func(b *blob.Bucket) error {
		fmt.Println(b.ID())
		return nil
	})
}

type cleanfn func() error

func tempdir() (string, cleanfn, error) {
	dir, err := os.MkdirTemp(os.TempDir(), "blob")
	if err != nil {
		return "", nil, err
	}
	return dir, func() error {
		return os.RemoveAll(dir)
	}, nil
}

Output:

sub-bucket-1
sub-bucket-2

func (*Bucket) New

func (b *Bucket) New(id string) (Blob, error)

New creates new blob in the Bucket

func (*Bucket) Open

func (b *Bucket) Open(id string) (Blob, error)

Open opens and existing Blob in the bucket

func (*Bucket) Path

func (b *Bucket) Path() string

Path returns path of the bucket

type ListBlobsFn

type ListBlobsFn = func(b Blob) error

ListBlobsFn is the callback function called by ListBlobsFn to list each blob for the given bucket returning error will cancel the walk

type ListBucketsFn

type ListBucketsFn = func(b *Bucket) error

ListBucketsFn is the callback function called by ListBuckets to list each sub bucket for the given bucket returning error will cancel the walk