compexperiments

README

compexperiments

experiments around compression

Documentation

Index

• Constants
• func AddTailConst[T PackType](v T)
• type BlockHeader
  • func (bh BlockHeader) AddGroup(bitlen, ntz int) BlockHeader
  • func (bh BlockHeader) BlockLen() int
  • func (bh BlockHeader) GetGroup(i int) (int, int)
  • func (bh BlockHeader) GroupCount() int
• type BytesSlice
  • func (bs *BytesSlice) Len() int
  • func (bs *BytesSlice) Reset()
  • func (bs *BytesSlice) Value(i int) []byte
  • func (bs *BytesSlice) Values(dst [][]byte) [][]byte
  • func (bs *BytesSlice) ValuesBytes() ([]byte, []int64)
• type CompressedBytesSlice
  • func (cs *CompressedBytesSlice) Add(src [][]byte, encoder any)
  • func (cs *CompressedBytesSlice) AddBytes(src []byte, offsets []int64, encoder any)
  • func (cs *CompressedBytesSlice) AddOne(src []byte, encoder any)
  • func (cs CompressedBytesSlice) Append(src [][]byte, encoder any) CompressedBytesSlice
  • func (cs CompressedBytesSlice) AppendBytes(src []byte, offsets []int64, encoder any) CompressedBytesSlice
  • func (cs CompressedBytesSlice) AppendOne(src []byte, encoder any) CompressedBytesSlice
  • func (cs *CompressedBytesSlice) BlockCount() int
  • func (cs *CompressedBytesSlice) BlockDataLen(iBlock int) int
  • func (cs *CompressedBytesSlice) BlockLen(iBlock int) int
  • func (cs *CompressedBytesSlice) BlockNum(i int) int
  • func (cs *CompressedBytesSlice) CompressedSize() int
  • func (cs *CompressedBytesSlice) DataLen() int
  • func (cs *CompressedBytesSlice) DeepCopy() (copy CompressedBytesSlice)
  • func (cs *CompressedBytesSlice) Export() ([]byte, []byte, []int64, int64, CompressedSlice[int64])
  • func (cs *CompressedBytesSlice) Get(i int, decoder any) []byte
  • func (cs *CompressedBytesSlice) GetAll(dst BytesSlice, decoder any) BytesSlice
  • func (cs *CompressedBytesSlice) GetBlock(dst BytesSlice, iBlock int, decoder any) (BytesSlice, int)
  • func (cs *CompressedBytesSlice) GetBlockBytes(dst []byte, dstOffsets []int64, iBlock int, decoder any) ([]byte, []int64, int)
  • func (cs *CompressedBytesSlice) GetBytes(dst []byte, dstOffsets []int64, decoder any) ([]byte, []int64)
  • func (cs *CompressedBytesSlice) Import(buf []byte, tail []byte, bufBlockOffsets []int64, lastOffset int64, ...)
  • func (cs *CompressedBytesSlice) IsBlockCompressed(iBlock int) bool
  • func (cs *CompressedBytesSlice) Len() int
  • func (cs *CompressedBytesSlice) MemSize() int
  • func (cs *CompressedBytesSlice) RemainingCapBytes() int
  • func (cs *CompressedBytesSlice) Truncate(i int, decoder any)
• type CompressedSlice
  • func (cs *CompressedSlice[T]) Add(src []T)
  • func (cs *CompressedSlice[T]) AddLossy(src []T, maxBits int)
  • func (cs *CompressedSlice[T]) AddOne(src T)
  • func (cs *CompressedSlice[T]) AddOneLossy(src T, maxBits int)
  • func (cs CompressedSlice[T]) Append(src []T) CompressedSlice[T]
  • func (cs CompressedSlice[T]) AppendLossy(src []T, maxBits int) CompressedSlice[T]
  • func (cs CompressedSlice[T]) AppendOne(src T) CompressedSlice[T]
  • func (cs CompressedSlice[T]) AppendOneLossy(src T, maxBits int) CompressedSlice[T]
  • func (cs *CompressedSlice[T]) BlockCount() int
  • func (cs *CompressedSlice[T]) BlockFirstValue(iBlock int) T
  • func (cs *CompressedSlice[T]) BlockLen(iBlock int) int
  • func (cs *CompressedSlice[T]) BlockMinMax(iBlock int) (T, T)
  • func (cs *CompressedSlice[T]) BlockNum(i int) int
  • func (cs *CompressedSlice[T]) CompressedSize() int
  • func (cs *CompressedSlice[T]) DeepCopy() (copy CompressedSlice[T])
  • func (cs *CompressedSlice[T]) Export() ([]uint64, []T, []int64, []minMax[T])
  • func (cs *CompressedSlice[T]) FirstValue() T
  • func (cs *CompressedSlice[T]) Get(i int) T
  • func (cs *CompressedSlice[T]) GetAll(dst []T) []T
  • func (cs *CompressedSlice[T]) GetBlock(dst []T, iBlock int) ([]T, int)
  • func (cs *CompressedSlice[T]) Import(buf []uint64, tail []T, blockOffsets []int64, minMax []minMax[T])
  • func (cs *CompressedSlice[T]) IsBlockCompressed(iBlock int) bool
  • func (cs *CompressedSlice[T]) LShiftBlocks(nBlocks int)
  • func (cs *CompressedSlice[T]) LastValue() T
  • func (cs *CompressedSlice[T]) Len() int
  • func (cs *CompressedSlice[T]) MemSize() int
  • func (cs *CompressedSlice[T]) RemainingCapBytes() int
  • func (cs *CompressedSlice[T]) Truncate(i int)
  • func (cs *CompressedSlice[T]) WithMinMax(v bool)
• type PackType

Constants

const MaxGroups = 4 * len(BlockHeader{})

Functions

func AddTailConst

func AddTailConst[T PackType](v T)

Types

type BlockHeader

type BlockHeader [2]uint64

BlockHeader is a uint64 used to store the number of bits in a group (7 bits) and the number of trailing zeros (6 bits). up to 4 groups can be stored in a block header. the number of groups are stored in the first 4 bits of the header.

func (BlockHeader) AddGroup

func (bh BlockHeader) AddGroup(bitlen, ntz int) BlockHeader

AddGroup adds a group of bits to the block header. bitlen is the number of bits in the group, and ntz is the number of trailing zeros. bitlen and ntz are stored in the header as 7 and 6-bit values. adding a group increases the number of groups in the header by 1.

func (BlockHeader) BlockLen

func (bh BlockHeader) BlockLen() int

BlockLen returns the number of compressed uint64 in the block not including the header.

func (BlockHeader) GetGroup

func (bh BlockHeader) GetGroup(i int) (int, int)

GetGroup returns the bitlen and ntz of the group at index i.

func (BlockHeader) GroupCount

func (bh BlockHeader) GroupCount() int

GroupCount returns the number of groups in the block header.

type BytesSlice

type BytesSlice struct {
	// contains filtered or unexported fields
}

func (*BytesSlice) Len

func (bs *BytesSlice) Len() int

func (*BytesSlice) Reset

func (bs *BytesSlice) Reset()

func (*BytesSlice) Value

func (bs *BytesSlice) Value(i int) []byte

func (*BytesSlice) Values

func (bs *BytesSlice) Values(dst [][]byte) [][]byte

func (*BytesSlice) ValuesBytes

func (bs *BytesSlice) ValuesBytes() ([]byte, []int64)

type CompressedBytesSlice

type CompressedBytesSlice struct {
	// contains filtered or unexported fields
}

func (*CompressedBytesSlice) Add

func (cs *CompressedBytesSlice) Add(src [][]byte, encoder any)

func (*CompressedBytesSlice) AddBytes

func (cs *CompressedBytesSlice) AddBytes(src []byte, offsets []int64, encoder any)

func (*CompressedBytesSlice) AddOne

func (cs *CompressedBytesSlice) AddOne(src []byte, encoder any)

func (CompressedBytesSlice) Append

func (cs CompressedBytesSlice) Append(src [][]byte, encoder any) CompressedBytesSlice

func (CompressedBytesSlice) AppendBytes

func (cs CompressedBytesSlice) AppendBytes(src []byte, offsets []int64, encoder any) CompressedBytesSlice

func (CompressedBytesSlice) AppendOne

func (cs CompressedBytesSlice) AppendOne(src []byte, encoder any) CompressedBytesSlice

func (*CompressedBytesSlice) BlockCount

func (cs *CompressedBytesSlice) BlockCount() int

func (*CompressedBytesSlice) BlockDataLen

func (cs *CompressedBytesSlice) BlockDataLen(iBlock int) int

func (*CompressedBytesSlice) BlockLen

func (cs *CompressedBytesSlice) BlockLen(iBlock int) int

func (*CompressedBytesSlice) BlockNum

func (cs *CompressedBytesSlice) BlockNum(i int) int

BlockNum returns the block (iBlock) given the index in the uncompressed slice

func (*CompressedBytesSlice) CompressedSize

func (cs *CompressedBytesSlice) CompressedSize() int

func (*CompressedBytesSlice) DataLen

func (cs *CompressedBytesSlice) DataLen() int

func (*CompressedBytesSlice) DeepCopy

func (cs *CompressedBytesSlice) DeepCopy() (copy CompressedBytesSlice)

DeepCopy returns a deep copy of the compressed bytes slice it is mainly used to be used in cunjunction with Truncate

func (*CompressedBytesSlice) Export

func (cs *CompressedBytesSlice) Export() ([]byte, []byte, []int64, int64, CompressedSlice[int64])

func (*CompressedBytesSlice) Get

func (cs *CompressedBytesSlice) Get(i int, decoder any) []byte

func (*CompressedBytesSlice) GetAll

func (cs *CompressedBytesSlice) GetAll(dst BytesSlice, decoder any) BytesSlice

func (*CompressedBytesSlice) GetBlock

func (cs *CompressedBytesSlice) GetBlock(dst BytesSlice, iBlock int, decoder any) (BytesSlice, int)

func (*CompressedBytesSlice) GetBlockBytes

func (cs *CompressedBytesSlice) GetBlockBytes(dst []byte, dstOffsets []int64, iBlock int, decoder any) ([]byte, []int64, int)

func (*CompressedBytesSlice) GetBytes

func (cs *CompressedBytesSlice) GetBytes(dst []byte, dstOffsets []int64, decoder any) ([]byte, []int64)

func (*CompressedBytesSlice) Import

func (cs *CompressedBytesSlice) Import(buf []byte, tail []byte, bufBlockOffsets []int64, lastOffset int64, offsets CompressedSlice[int64])

func (*CompressedBytesSlice) IsBlockCompressed

func (cs *CompressedBytesSlice) IsBlockCompressed(iBlock int) bool

func (*CompressedBytesSlice) Len

func (cs *CompressedBytesSlice) Len() int

func (*CompressedBytesSlice) MemSize

func (cs *CompressedBytesSlice) MemSize() int

func (*CompressedBytesSlice) RemainingCapBytes

func (cs *CompressedBytesSlice) RemainingCapBytes() int

func (*CompressedBytesSlice) Truncate

func (cs *CompressedBytesSlice) Truncate(i int, decoder any)

type CompressedSlice

type CompressedSlice[T PackType] struct {
	// contains filtered or unexported fields
}

func (*CompressedSlice[T]) Add

func (cs *CompressedSlice[T]) Add(src []T)

func (*CompressedSlice[T]) AddLossy

func (cs *CompressedSlice[T]) AddLossy(src []T, maxBits int)

func (*CompressedSlice[T]) AddOne

func (cs *CompressedSlice[T]) AddOne(src T)

func (*CompressedSlice[T]) AddOneLossy

func (cs *CompressedSlice[T]) AddOneLossy(src T, maxBits int)

func (CompressedSlice[T]) Append

func (cs CompressedSlice[T]) Append(src []T) CompressedSlice[T]

func (CompressedSlice[T]) AppendLossy

func (cs CompressedSlice[T]) AppendLossy(src []T, maxBits int) CompressedSlice[T]

func (CompressedSlice[T]) AppendOne

func (cs CompressedSlice[T]) AppendOne(src T) CompressedSlice[T]

func (CompressedSlice[T]) AppendOneLossy

func (cs CompressedSlice[T]) AppendOneLossy(src T, maxBits int) CompressedSlice[T]

func (*CompressedSlice[T]) BlockCount

func (cs *CompressedSlice[T]) BlockCount() int

func (*CompressedSlice[T]) BlockFirstValue

func (cs *CompressedSlice[T]) BlockFirstValue(iBlock int) T

func (*CompressedSlice[T]) BlockLen

func (cs *CompressedSlice[T]) BlockLen(iBlock int) int

func (*CompressedSlice[T]) BlockMinMax

func (cs *CompressedSlice[T]) BlockMinMax(iBlock int) (T, T)

func (*CompressedSlice[T]) BlockNum

func (cs *CompressedSlice[T]) BlockNum(i int) int

BlockNum returns the block (iBlock) given the index in the uncompressed slice

func (*CompressedSlice[T]) CompressedSize

func (cs *CompressedSlice[T]) CompressedSize() int

func (*CompressedSlice[T]) DeepCopy

func (cs *CompressedSlice[T]) DeepCopy() (copy CompressedSlice[T])

DeepCopy returns a deep copy of the CompressedSlice mainly designed to be used in conjunction Truncate

func (*CompressedSlice[T]) Export

func (cs *CompressedSlice[T]) Export() ([]uint64, []T, []int64, []minMax[T])

func (*CompressedSlice[T]) FirstValue

func (cs *CompressedSlice[T]) FirstValue() T

func (*CompressedSlice[T]) Get

func (cs *CompressedSlice[T]) Get(i int) T

func (*CompressedSlice[T]) GetAll

func (cs *CompressedSlice[T]) GetAll(dst []T) []T

func (*CompressedSlice[T]) GetBlock

func (cs *CompressedSlice[T]) GetBlock(dst []T, iBlock int) ([]T, int)

func (*CompressedSlice[T]) Import

func (cs *CompressedSlice[T]) Import(buf []uint64, tail []T, blockOffsets []int64, minMax []minMax[T])

func (*CompressedSlice[T]) IsBlockCompressed

func (cs *CompressedSlice[T]) IsBlockCompressed(iBlock int) bool

func (*CompressedSlice[T]) LShiftBlocks

func (cs *CompressedSlice[T]) LShiftBlocks(nBlocks int)

func (*CompressedSlice[T]) LastValue

func (cs *CompressedSlice[T]) LastValue() T

func (*CompressedSlice[T]) Len

func (cs *CompressedSlice[T]) Len() int

func (*CompressedSlice[T]) MemSize

func (cs *CompressedSlice[T]) MemSize() int

func (*CompressedSlice[T]) RemainingCapBytes

func (cs *CompressedSlice[T]) RemainingCapBytes() int

func (*CompressedSlice[T]) Truncate

func (cs *CompressedSlice[T]) Truncate(i int)

func (*CompressedSlice[T]) WithMinMax

func (cs *CompressedSlice[T]) WithMinMax(v bool)

type PackType

type PackType interface {
	~int | ~int32 | ~uint32 | ~int64 | ~uint64 | ~float32 | ~float64
}