Version: v4.15.0+incompatible Latest Latest

This package is not in the latest version of its module.

Go to latest
Published: Sep 17, 2018 License: GPL-3.0 Imports: 20 Imported by: 0




View Source
const (
	VSIZE           = 1024
	KFACTOR         = 64
	VBSIZE          = 2 + 9*VSIZE + 9*VSIZE + 2*VSIZE //Worst case with huffman
	CBSIZE          = 1 + KFACTOR*9*6
	DBSIZE          = VBSIZE
	PWFACTOR        = uint8(6) //1<<6 == 64
	RELOCATION_BASE = 0xFF00000000000000
	SUPERBLOCK_SIZE = 8 + 8 // address + walltime

Note to self, if you bump VSIZE such that the max blob goes past 2^16, make sure to adapt providers

View Source
const ABSZERO = 1
View Source
const FULLZERO = 2
View Source
const FlagsMask uint8 = 3
View Source
const LatestGeneration = uint64(^(uint64(0)))
View Source

Like roughly 1MB

View Source

If the superblock cache exceeds the max above, randomly delete this many elements from it

View Source
const VALUE = 0

These functions allow us to read/write the packed numbers in the datablocks These are huffman encoded in big endian 0xxx xxxx 7 0x00 10xx xxxx +1 14 0x80 1100 xxxx +2 20 0xC0 1101 xxxx +3 28 0xD0 1110 xxxx +4 36 0xE0 1111 00xx +5 42 0xF0 1111 01xx +6 50 0xF4 1111 10xx +7 58 0xF8 1111 1100 +8 64 0xFC 1111 1101 +0 ABSZERO (special symbol) 0xFD 1111 1110 +0 FULLZERO (special symbol) 0xFE


View Source
var ErrDatablockNotFound = errors.New("Coreblock not found")
View Source
var ErrGenerationNotFound = errors.New("Generation not found")


func CreateDatabase

func CreateDatabase(cfg configprovider.Configuration, overwrite bool)

func LinkAndStore

func LinkAndStore(uuid []byte, bs *BlockStore, bp bprovider.StorageProvider, vblocks []*Vectorblock, cblocks []*Coreblock) map[uint64]uint64

func UUIDToMapKey

func UUIDToMapKey(id uuid.UUID) [16]byte


type BlockStore

type BlockStore struct {
	// contains filtered or unexported fields

func NewBlockStore

func NewBlockStore(cfg configprovider.Configuration, rm *rez.RezManager) (*BlockStore, error)

func (bs *BlockStore) UnlinkGenerations(id uuid.UUID, sgen uint64, egen uint64) error {

iter := bs.db.C("superblocks").Find(bson.M{"uuid": id.String(), "gen": bson.M{"$gte": sgen, "$lt": egen}, "unlinked": false}).Iter()
rs := fake_sblock{}
for iter.Next(&rs) {
	rs.Unlinked = true
	_, err := bs.db.C("superblocks").Upsert(bson.M{"uuid": id.String(), "gen": rs.Gen}, rs)
	if err != nil {
return nil


func (*BlockStore) FlushSuperblockFromCache

func (bs *BlockStore) FlushSuperblockFromCache(uu uuid.UUID)

func (*BlockStore) FreeCoreblock

func (bs *BlockStore) FreeCoreblock(cb **Coreblock)

func (*BlockStore) FreeVectorblock

func (bs *BlockStore) FreeVectorblock(vb **Vectorblock)

func (*BlockStore) LASMetrics

func (bs *BlockStore) LASMetrics(m *LASMetric)

func (*BlockStore) LoadSuperblock

func (bs *BlockStore) LoadSuperblock(ctx context.Context, id uuid.UUID, generation uint64) (*Superblock, bte.BTE)

func (*BlockStore) LoadSuperblockFromCache

func (bs *BlockStore) LoadSuperblockFromCache(uu uuid.UUID) *Superblock

func (*BlockStore) NotifyWriteLockLost

func (bs *BlockStore) NotifyWriteLockLost()

This is called if our write lock changes. Need to invalidate caches

func (*BlockStore) ObtainGeneration

func (bs *BlockStore) ObtainGeneration(ctx context.Context, id uuid.UUID) (*Generation, bte.BTE)

* This obtains a generation, blocking if necessary

func (*BlockStore) PutSuperblockInCache

func (bs *BlockStore) PutSuperblockInCache(s *Superblock)

func (*BlockStore) ReadDatablock

func (bs *BlockStore) ReadDatablock(ctx context.Context, uuid uuid.UUID, addr uint64, impl_Generation uint64, impl_Pointwidth uint8, impl_StartTime int64) (Datablock, bte.BTE)

func (*BlockStore) StorageProvider

func (bs *BlockStore) StorageProvider() bprovider.StorageProvider

func (*BlockStore) StreamExists

func (bs *BlockStore) StreamExists(ctx context.Context, id uuid.UUID) (bool, bte.BTE)

type BlockType

type BlockType uint64
const (
	Vector BlockType = 1
	Core   BlockType = 2
	Bad    BlockType = 255

func DatablockGetBufferType

func DatablockGetBufferType(buf []byte) BlockType

type CacheItem

type CacheItem struct {
	// contains filtered or unexported fields

type Coreblock

type Coreblock struct {

	//Metadata, not copied
	Identifier uint64 "metadata,implicit"
	Generation uint64 "metadata,implicit"

	//Payload, copied
	PointWidth  uint8 "implicit"
	StartTime   int64 "implicit"
	Addr        [KFACTOR]uint64
	Count       [KFACTOR]uint64
	Min         [KFACTOR]float64
	Mean        [KFACTOR]float64
	Max         [KFACTOR]float64
	CGeneration [KFACTOR]uint64

func (*Coreblock) CopyInto

func (src *Coreblock) CopyInto(dst *Coreblock)

Copy a core block, only copying the payload, not the metadata

func (*Coreblock) Deserialize

func (c *Coreblock) Deserialize(src []byte)

func (*Coreblock) GetDatablockType

func (*Coreblock) GetDatablockType() BlockType

func (*Coreblock) Serialize

func (c *Coreblock) Serialize(dst []byte) []byte

type Datablock

type Datablock interface {
	GetDatablockType() BlockType

type Generation

type Generation struct {
	Cur_SB *Superblock
	New_SB *Superblock
	// contains filtered or unexported fields

A generation stores all the information acquired during a write pass. * A superblock contains all the information required to navigate a tree.

func (*Generation) AllocateCoreblock

func (gen *Generation) AllocateCoreblock() *Coreblock

func (*Generation) AllocateVectorblock

func (gen *Generation) AllocateVectorblock() *Vectorblock

func (*Generation) Commit

func (gen *Generation) Commit() (map[uint64]uint64, bte.BTE)

The returned address map is primarily for unit testing

func (*Generation) HintEvictReplaced

func (g *Generation) HintEvictReplaced(addr uint64)

func (*Generation) Number

func (g *Generation) Number() uint64

func (*Generation) UpdateRootAddr

func (g *Generation) UpdateRootAddr(addr uint64)

func (*Generation) Uuid

func (g *Generation) Uuid() *uuid.UUID

type LASMetric

type LASMetric struct {
	// contains filtered or unexported fields

type Superblock

type Superblock struct {
	// contains filtered or unexported fields

func DeserializeSuperblock

func DeserializeSuperblock(id uuid.UUID, gen uint64, arr []byte) *Superblock

func NewSuperblock

func NewSuperblock(id uuid.UUID) *Superblock

func (*Superblock) CloneInc

func (s *Superblock) CloneInc() *Superblock

func (*Superblock) Gen

func (s *Superblock) Gen() uint64

func (*Superblock) Root

func (s *Superblock) Root() uint64

func (*Superblock) Serialize

func (s *Superblock) Serialize() []byte

func (*Superblock) Uuid

func (s *Superblock) Uuid() uuid.UUID

type Vectorblock

type Vectorblock struct {

	//Metadata, not copied on clone
	Identifier uint64 "metadata,implicit"
	Generation uint64 "metadata,implicit"

	//Payload, copied on clone
	Len        uint16
	PointWidth uint8 "implicit"
	StartTime  int64 "implicit"
	Time       [VSIZE]int64
	Value      [VSIZE]float64

The leaf datablock type. The tags allow unit tests to work out if clone / serdes are working properly metadata is not copied when a node is cloned implicit is not serialised

func (*Vectorblock) CopyInto

func (src *Vectorblock) CopyInto(dst *Vectorblock)

func (*Vectorblock) Deserialize

func (v *Vectorblock) Deserialize(src []byte)

func (*Vectorblock) GetDatablockType

func (*Vectorblock) GetDatablockType() BlockType

func (*Vectorblock) Serialize

func (v *Vectorblock) Serialize(dst []byte) []byte

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL