core

Documentation

Index

• Constants
• Variables
• func Ext(filename string) (ext string)
• func HeaderSize(extsize uint64) (size uint64)
• func NeedleMarshal(n *Needle) (data []byte, err error)
• type Directory
• type Iterator
• type LeveldbDirectory
  • func NewLeveldbDirectory(dir string) (d *LeveldbDirectory, err error)
  • func (d *LeveldbDirectory) Del(id uint64) (err error)
  • func (d *LeveldbDirectory) Get(id uint64) (n *Needle, err error)
  • func (d *LeveldbDirectory) Has(id uint64) (has bool)
  • func (d *LeveldbDirectory) Iter() (iter Iterator)
  • func (d *LeveldbDirectory) New(n *Needle) (err error)
  • func (d *LeveldbDirectory) Set(id uint64, needle *Needle) (err error)
• type LeveldbIterator
  • func (it *LeveldbIterator) Next() (key []byte, exists bool)
  • func (it *LeveldbIterator) Release()
• type Needle
  • func NeedleUnmarshal(b []byte) (n *Needle, err error)
  • func (n *Needle) Read(b []byte) (num int, err error)
  • func (n *Needle) Write(b []byte) (num int, err error)
• type Volume
  • func NewVolume(id uint64, dir string) (v *Volume, err error)
  • func (v *Volume) CheckCurrentIndex() (same bool)
  • func (v *Volume) DelNeedle(id uint64) (err error)
  • func (v *Volume) Fragment() (err error)
  • func (v *Volume) GetFile(id uint64) (data []byte, ext string, err error)
  • func (v *Volume) GetNeedle(id uint64) (n *Needle, err error)
  • func (v *Volume) NewFile(data []byte, filename string) (id uint64, err error)
  • func (v *Volume) NewNeedle(id uint64, data []byte, filename string) (n *Needle, err error)
  • func (v *Volume) Print()
  • func (v *Volume) ReadHeader(offset int64) (header []byte, err error)
  • func (v *Volume) RemainingSpace() (size uint64)

Constants

const (
	TruncateSize  uint64 = 1 << 30            //1GB
	MaxVolumeSize uint64 = 128 * TruncateSize // 128GB
	InitIndexSize uint64 = 8
	DefaultDir    string = "/tmp/fs"
)

Variables

var (
	ErrNilNeedle     = errors.New("Nil needle")
	ErrWrongLen      = errors.New("Wrong length of needle bytes ")
	ErrLeakSpace     = errors.New("Volume leak of space")
	ErrDeleted       = errors.New("Needle is deleted")
	ErrSmallNeedle   = errors.New("Needle's size less than data size")
	ErrWrongCheckSum = errors.New("Checksum err")
)

var NeedleFixSize uint64 = 44 // 不包括 len(Filename)

64 * 3 + 32 + 64*2 = = 44 bytes for an needle header

Functions

func Ext

func Ext(filename string) (ext string)

从完整文件名中获取扩展名

func HeaderSize

func HeaderSize(extsize uint64) (size uint64)

HeaderSize Needle 中除了正文外的额外信息的大小

func NeedleMarshal

func NeedleMarshal(n *Needle) (data []byte, err error)

NeedleMarshal: Needle struct -> bytes

Types

type Directory

type Directory interface {
	Get(id uint64) (n *Needle, err error)
	New(n *Needle) (err error)
	Has(id uint64) (has bool)
	Del(id uint64) (err error)
	Set(id uint64, n *Needle) (err error)
	Iter() (iter Iterator)
}

type Iterator

type Iterator interface {
	Next() (key []byte, exists bool)
	Release()
}

type LeveldbDirectory

type LeveldbDirectory struct {
	// contains filtered or unexported fields
}

func NewLeveldbDirectory

func NewLeveldbDirectory(dir string) (d *LeveldbDirectory, err error)

func (*LeveldbDirectory) Del

func (d *LeveldbDirectory) Del(id uint64) (err error)

func (*LeveldbDirectory) Get

func (d *LeveldbDirectory) Get(id uint64) (n *Needle, err error)

func (*LeveldbDirectory) Has

func (d *LeveldbDirectory) Has(id uint64) (has bool)

func (*LeveldbDirectory) Iter

func (d *LeveldbDirectory) Iter() (iter Iterator)

func (*LeveldbDirectory) New

func (d *LeveldbDirectory) New(n *Needle) (err error)

func (*LeveldbDirectory) Set

func (d *LeveldbDirectory) Set(id uint64, needle *Needle) (err error)

type LeveldbIterator

type LeveldbIterator struct {
	// contains filtered or unexported fields
}

func (*LeveldbIterator) Next

func (it *LeveldbIterator) Next() (key []byte, exists bool)

func (*LeveldbIterator) Release

func (it *LeveldbIterator) Release()

type Needle

type Needle struct {
	ID       uint64 // 唯一ID， 64
	Size     uint64 // size of BODY
	Offset   uint64 // points to start of header
	File     *os.File
	FileExt  string // 文件扩展名, 下载时用来判断 content-type
	Checksum uint32

	CreatedAt time.Time
	UpdatedAt time.Time
	// contains filtered or unexported fields
}

Needle in Haystack

func NeedleUnmarshal

func NeedleUnmarshal(b []byte) (n *Needle, err error)

NeedleUnmarshal: bytes -> needle struct

func (*Needle) Read

func (n *Needle) Read(b []byte) (num int, err error)

func (*Needle) Write

func (n *Needle) Write(b []byte) (num int, err error)

type Volume

type Volume struct {
	ID            uint64
	File          *os.File
	Directory     Directory
	Size          uint64
	Path          string
	CurrentOffset uint64 // 从文件中恢复的时候， 第一个byte就是currentoffset. 每次写入文件也要更新此offset。
	// contains filtered or unexported fields
}

TODO Compression TODO Several volumes make up a Store

func NewVolume

func NewVolume(id uint64, dir string) (v *Volume, err error)

func (*Volume) CheckCurrentIndex

func (v *Volume) CheckCurrentIndex() (same bool)

func (*Volume) DelNeedle

func (v *Volume) DelNeedle(id uint64) (err error)

func (*Volume) Fragment

func (v *Volume) Fragment() (err error)

func (*Volume) GetFile

func (v *Volume) GetFile(id uint64) (data []byte, ext string, err error)

func (*Volume) GetNeedle

func (v *Volume) GetNeedle(id uint64) (n *Needle, err error)

func (*Volume) NewFile

func (v *Volume) NewFile(data []byte, filename string) (id uint64, err error)

func (*Volume) NewNeedle

func (v *Volume) NewNeedle(id uint64, data []byte, filename string) (n *Needle, err error)

NewNeedle allocate a new needle.

func (*Volume) Print

func (v *Volume) Print()

func (*Volume) ReadHeader

func (v *Volume) ReadHeader(offset int64) (header []byte, err error)

func (*Volume) RemainingSpace

func (v *Volume) RemainingSpace() (size uint64)