needle

package
v0.0.0-...-e27f3f3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 2, 2019 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	HeaderSize            = 16 //should never change this
	PaddingSize           = 8
	ChecksumSize          = 4
	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
	TombstoneFileSize     = math.MaxUint32
	PairNamePrefix        = "Seaweed-"
	IndexSize             = 16

	FlagGzip                = 0x01
	FlagHasName             = 0x02
	FlagHasMime             = 0x04
	FlagHasLastModifiedDate = 0x08
	FlagHasTtl              = 0x10
	FlagHasPairs            = 0x20
	FlagIsChunkManifest     = 0x80
	LastModifiedBytesLength = 5
	TtlBytesLength          = 2
)
View Source 
const (
	RowsToRead = 1024
)

Variables

View Source 
var (
	EnableBytesCache = true
)

Functions

func IdxFileEntry 

func IdxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32)

func ParseKeyHash 

func ParseKeyHash(key_hash_string string) (uint64, uint32, error)

func ParseUpload 

func ParseUpload(r *http.Request) (
	fileName string, data []byte,
	mimeType string, pairMap map[string]string,
	isGzipped bool, originalDataSize int,
	modifiedTime uint64,
	TL *ttl.TTL,
	e error)

func WalkIndexFile 

func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error

walks through the index file, calls fn function with each key, offset, size stops with the error returned by the fn function

Types

type Block 

type Block struct {
	Bytes []byte
	// contains filtered or unexported fields
}

type CompactMap 

type CompactMap struct {
	// contains filtered or unexported fields
}

This map assumes mostly inserting increasing keys

func NewCompactMap 

func NewCompactMap() *CompactMap

func (*CompactMap) Delete 

func (cm *CompactMap) Delete(key Key) uint32

func (*CompactMap) Get 

func (cm *CompactMap) Get(key Key) (*NeedleValue, bool)

func (*CompactMap) Set 

func (cm *CompactMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)

func (*CompactMap) Visit 

func (cm *CompactMap) Visit(visit func(NeedleValue) error) error

Visit visits all entries or stop if any error when visiting

type CompactSection 

type CompactSection struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewCompactSection 

func NewCompactSection(start Key) *CompactSection

func (*CompactSection) Delete 

func (cs *CompactSection) Delete(key Key) uint32

return old entry size

func (*CompactSection) Get 

func (cs *CompactSection) Get(key Key) (*NeedleValue, bool)

func (*CompactSection) Set 

func (cs *CompactSection) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)

return old entry size

type Key 

type Key uint64

type Mapper 

type Mapper interface {
	Put(key uint64, offset uint32, size uint32) error
	Get(key uint64) (element *NeedleValue, ok bool)
	Delete(key uint64, offset uint32) error
	Close()
	Destroy() error
	ContentSize() uint64
	DeletedSize() uint64
	FileCount() int
	DeletedCount() int
	MaxFileKey() uint64
	IndexFileSize() uint64
	IndexFileContent() ([]byte, error)
	IndexFileName() string
}

type Needle 

type Needle struct {
	Cookie       uint32 `comment:"random number to mitigate brute force lookups"`
	Id           uint64 `comment:"needle id"`
	Size         uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
	DataSize     uint32 `comment:"Data size"` //version2
	Data         []byte `comment:"The actual file data"`
	Flags        byte   `comment:"boolean flags"` //version2
	NameSize     uint8  //version2
	Name         []byte `comment:"maximum 256 characters"` //version2
	MimeSize     uint8  //version2
	Mime         []byte `comment:"maximum 256 characters"` //version2
	PairsSize    uint16 //version2
	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
	Ttl          *ttl.TTL
	Checksum     crc.CRC `comment:"CRC32 to check integrity"`
	Padding      []byte  `comment:"Aligned to 8 bytes"`
	// contains filtered or unexported fields
}

* A Needle means a uploaded and stored file. * Needle file size is limited to 4GB for now.

func NewNeedle 

func NewNeedle(r *http.Request) (n *Needle, originalSize int, e error)

func (*Needle) Append 

func (n *Needle) Append(w io.Writer, vers version.Version) (size uint32, actualSize int64, err error)

func (*Needle) Etag 

func (n *Needle) Etag() string

func (*Needle) HasLastModifiedDate 

func (n *Needle) HasLastModifiedDate() bool

func (*Needle) HasMime 

func (n *Needle) HasMime() bool

func (*Needle) HasName 

func (n *Needle) HasName() bool

func (*Needle) HasPairs 

func (n *Needle) HasPairs() bool

func (*Needle) HasTtl 

func (n *Needle) HasTtl() bool

func (*Needle) IsGzipped 

func (n *Needle) IsGzipped() bool

func (*Needle) ParseHeader 

func (n *Needle) ParseHeader(byte []byte)

func (*Needle) ParsePath 

func (n *Needle) ParsePath(fid string) (err error)

func (*Needle) ReadData 

func (n *Needle) ReadData(r *os.File, offset int64, size uint32, vers version.Version) (err error)

func (*Needle) ReleaseMemory 

func (n *Needle) ReleaseMemory()

func (*Needle) SetGzipped 

func (n *Needle) SetGzipped()

func (*Needle) SetHasLastModifiedDate 

func (n *Needle) SetHasLastModifiedDate()

func (*Needle) SetHasMime 

func (n *Needle) SetHasMime()

func (*Needle) SetHasName 

func (n *Needle) SetHasName()

func (*Needle) SetHasPairs 

func (n *Needle) SetHasPairs()

func (*Needle) SetHasTtl 

func (n *Needle) SetHasTtl()

type NeedleMap 

type NeedleMap struct {
	// contains filtered or unexported fields
}

func LoadCompactNeedleMap 

func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error)

func NewCompactNeedleMap 

func NewCompactNeedleMap(file *os.File) *NeedleMap

func (*NeedleMap) Close 

func (nm *NeedleMap) Close()

func (*NeedleMap) Delete 

func (nm *NeedleMap) Delete(key uint64, offset uint32) error

func (*NeedleMap) Destroy 

func (nm *NeedleMap) Destroy() error

func (*NeedleMap) Get 

func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool)

func (*NeedleMap) IndexFileContent 

func (nm *NeedleMap) IndexFileContent() ([]byte, error)

func (*NeedleMap) IndexFileName 

func (nm *NeedleMap) IndexFileName() string

func (*NeedleMap) IndexFileSize 

func (nm *NeedleMap) IndexFileSize() uint64

func (*NeedleMap) Put 

func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error

type NeedleValue 

type NeedleValue struct {
	Key    Key
	Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
	Size   uint32 `comment:"Size of the data portion"`
}

type NeedleValueMap 

type NeedleValueMap interface {
	Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)
	Delete(key Key) uint32
	Get(key Key) (*NeedleValue, bool)
	Visit(visit func(NeedleValue) error) error
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.