blkidx

README

blkidx

a file checksum indexer.
after the initial indexing stage duplicate files can be found and removed.

Get blkidx:

go get github.com/phicode/blkidx/cmd/blkidx

Documentation

Index

• Variables
• func HashAll(r io.Reader, algorithm crypto.Hash, blockSize int) (all []byte, blocks [][]byte, n int64, err error)
• type Blob
  • func IndexFile(name string, config IndexConfig) (blob *Blob, err error)
  • func (b *Blob) CheckOptimisticLock(update *Blob) error
  • func (b *Blob) EqualHash(other *Blob) bool
  • func (b *Blob) HasChanged(size int64, mtime time.Time) bool
  • func (b *Blob) Validate() error
• type EqualBlobs
  • func (e *EqualBlobs) Append(blob *Blob)
  • func (e *EqualBlobs) AppendRaw(name string, size int64)
  • func (e *EqualBlobs) ContainsAnyName(names map[string]struct{}) bool
• type Hasher
  • func NewHasher(algorithm crypto.Hash, blockSize int) Hasher
• type Index
  • func NewMemoryIndex() Index
  • func NewSqlIndex(db *sql.DB) (Index, error)
• type IndexConfig
• type Indexer
  • func (i *Indexer) IndexAll(c <-chan *fs.PathElem)
• type LockedIndex
  • func (i *LockedIndex) AllNames() (Names, error)
  • func (i *LockedIndex) Count() (int, error)
  • func (i *LockedIndex) FindEqualHashes() ([]EqualBlobs, error)
  • func (i *LockedIndex) LookupByName(name string) (*Blob, error)
  • func (i *LockedIndex) Remove(names Names) error
  • func (i *LockedIndex) Store(blob *Blob) error
• type Names
  • func (n Names) Sort()
• type OptimisticLockingError
  • func (o *OptimisticLockingError) Error() string

Variables

var (
	DefaultHashAlgorithm crypto.Hash = crypto.SHA1
	DefaultHashBlockSize int         = 64 << 20
)

Functions

func HashAll

func HashAll(r io.Reader, algorithm crypto.Hash, blockSize int) (all []byte, blocks [][]byte, n int64, err error)

Types

type Blob

type Blob struct {
	Name      string
	Version   uint64
	IndexTime time.Time

	Size    int64
	ModTime time.Time

	HashAlgorithm crypto.Hash

	// hash of the full blob
	Hash []byte

	// size of hashed blocks
	HashBlockSize int

	// hashes of individual blocks
	HashedBlocks [][]byte
}

func IndexFile

func IndexFile(name string, config IndexConfig) (blob *Blob, err error)

func (*Blob) CheckOptimisticLock

func (b *Blob) CheckOptimisticLock(update *Blob) error

func (*Blob) EqualHash

func (b *Blob) EqualHash(other *Blob) bool

func (*Blob) HasChanged

func (b *Blob) HasChanged(size int64, mtime time.Time) bool

func (*Blob) Validate

func (b *Blob) Validate() error

type EqualBlobs

type EqualBlobs struct {
	Names Names
	Size  int64
}

func (*EqualBlobs) Append

func (e *EqualBlobs) Append(blob *Blob)

func (*EqualBlobs) AppendRaw

func (e *EqualBlobs) AppendRaw(name string, size int64)

func (*EqualBlobs) ContainsAnyName

func (e *EqualBlobs) ContainsAnyName(names map[string]struct{}) bool

type Hasher

type Hasher interface {
	io.Writer

	// the first return value is the hash of all data that has been written to this Hasher.
	// the second return value are the hashes of each block that has been written to this Hasher.
	Finish() ([]byte, [][]byte)
}

func NewHasher

func NewHasher(algorithm crypto.Hash, blockSize int) Hasher

type Index

type Index interface {
	// stores a blob by its name.
	// if the version field is zero and no such blob exists it will be stored.
	// existing blobs will be overwritten if the new version is excalty one higher than the existing one.
	// otherwise a OptimisticLockingError will be returned.
	Store(blob *Blob) error

	// blob lookup by name. if no blob by a certain name exists "nil, nil" is returned.
	// the error return value is indicative of problems with the underlying storage strategy.
	LookupByName(name string) (*Blob, error)

	FindEqualHashes() ([]EqualBlobs, error)

	AllNames() (Names, error)

	Remove(names Names) error

	Count() (int, error)
}

func NewMemoryIndex

func NewMemoryIndex() Index

func NewSqlIndex

func NewSqlIndex(db *sql.DB) (Index, error)

type IndexConfig

type IndexConfig struct {
	HashAlgorithm crypto.Hash
	BlockSizes    int
}

type Indexer

type Indexer struct {
	Index Index

	Log *log.Logger

	Concurrency int
	// contains filtered or unexported fields
}

func (*Indexer) IndexAll

func (i *Indexer) IndexAll(c <-chan *fs.PathElem)

type LockedIndex

type LockedIndex struct {
	Backend Index
	// contains filtered or unexported fields
}

func (*LockedIndex) AllNames

func (i *LockedIndex) AllNames() (Names, error)

func (*LockedIndex) Count

func (i *LockedIndex) Count() (int, error)

func (*LockedIndex) FindEqualHashes

func (i *LockedIndex) FindEqualHashes() ([]EqualBlobs, error)

func (*LockedIndex) LookupByName

func (i *LockedIndex) LookupByName(name string) (*Blob, error)

func (*LockedIndex) Remove

func (i *LockedIndex) Remove(names Names) error

func (*LockedIndex) Store

func (i *LockedIndex) Store(blob *Blob) error

type Names

type Names []string

func (Names) Sort

func (n Names) Sort()

type OptimisticLockingError

type OptimisticLockingError struct {
	Name          string
	FailedVersion uint64
}

func (*OptimisticLockingError) Error

func (o *OptimisticLockingError) Error() string