jengablk

package
v0.0.5-alpha Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 2, 2021 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	BlkFileMagicCode       uint32 = 0x58466AFB
	BlkFileHeadSize               = 10
	BlkFileBufferSize             = 32 * 1024
	BlkHeaderUnknownOffset        = -1
)
View Source 
const (
	BlkFileV2Version uint16 = 0x0002
)
View Source 
const (
	BlkFileVersion uint16 = 0x0001
)
View Source 
const (
	MaxVarUintBufSize = 10
)

Variables

View Source 
var BlkFileOpeners blkFileOpeners
View Source 
var BlkFileV2Openers blkFileV2Openers
View Source 
var BlockV1Opts blockV1Opts
View Source 
var BlockV2Opts blockV2Opts

Functions

func CalcVaruintLen 

func CalcVaruintLen(x uint64) int

func DecodeVarint 

func DecodeVarint(buf []byte) int64

func DecodeVaruint 

func DecodeVaruint(buf []byte) uint64

func EncodeVaruint 

func EncodeVaruint(buf []byte, x uint64) int

Base 128 Varint的介绍：https://developers.google.com/protocol-buffers/docs/encoding Base 128 Varint，为什么叫128？其实，就是因为只采用7bit的空间来存储有效数据，7bit当然最大只能存储128了。 常规的一个byte是8个bit位，但在Base 128 Varint编码中，将最高的第8位用来作为一个标志位， 如果这一位是1，就表示这个字节后面还有其他字节，如果这个位是0的话，就表示这是最后一个字节了， 这样一来，就可以准确的知道一个整数的结束位置了。

func GetFileSize 

func GetFileSize(s string) int64

func NewV1BlockFile 

func NewV1BlockFile(path string, opts ...BlocksV1Opt) *blockV1

func NewV1Blocks 

func NewV1Blocks(opts ...BlocksV1Opt) *blockV1

func NewV2BlockFile 

func NewV2BlockFile(path string, opts ...BlocksV2Opt) *blockV2

func NewV2Blocks 

func NewV2Blocks(opts ...BlocksV2Opt) *blockV2

func WriteFileHeader 

func WriteFileHeader(h FileHeader, w io.Writer) error

Types

type BlkFile 

type BlkFile struct {
	// contains filtered or unexported fields
}

File format: |MAGIC NUNMBER(2 Bytes)|VERSION(2 Bytes)|DATA FORMAT(2 Bytes)|REVERSE(2 Bytes)|ENTITY_1|ENTITY_2|...|ENTITY_N| Entity format: |VARINT(1-10 Bytes)|STRING(string length)|VARINT(1-10 Bytes)|DATA(data size)|

func NewBlkFile 

func NewBlkFile(path string) *BlkFile

func NewBlkFileWithOpener 

func NewBlkFileWithOpener(opener Opener) *BlkFile

func (*BlkFile) Close 

func (bf *BlkFile) Close() error

func (*BlkFile) Flush 

func (bf *BlkFile) Flush() error

func (*BlkFile) Open 

func (bf *BlkFile) Open(flag flags.OpenFlag) error

func (*BlkFile) ReadBlock 

func (bf *BlkFile) ReadBlock(w io.Writer) (*BlkHeader, error)

func (*BlkFile) ReadFile 

func (bf *BlkFile) ReadFile(path string) (*BlkHeader, error)

func (*BlkFile) WriteBlock 

func (bf *BlkFile) WriteBlock(header *BlkHeader, reader io.Reader) (int64, error)

func (*BlkFile) WriteFile 

func (bf *BlkFile) WriteFile(path string) (int64, error)

type BlkFileV2 

type BlkFileV2 struct {
	// contains filtered or unexported fields
}

File format: |MAGIC NUNMBER(2 Bytes)|VERSION(2 Bytes)|DATA FORMAT(2 Bytes)|REVERSE(2 Bytes)|ENTITY_1|ENTITY_2|...|ENTITY_N| Entity format: |VARINT(1-10 Bytes)|STRING(string length)|DATA SIZE(8 Bytes)|DATA(data size)|

func NewBlkFileV2 

func NewBlkFileV2(path string) *BlkFileV2

func NewBlkFileV2WithOpener 

func NewBlkFileV2WithOpener(opener Opener) *BlkFileV2

func (*BlkFileV2) Close 

func (bf *BlkFileV2) Close() error

func (*BlkFileV2) Flush 

func (bf *BlkFileV2) Flush() error

func (*BlkFileV2) Open 

func (bf *BlkFileV2) Open(flag flags.OpenFlag) error

func (*BlkFileV2) ReadBlock 

func (bf *BlkFileV2) ReadBlock(w io.Writer) (*BlkHeader, error)

func (*BlkFileV2) ReadFile 

func (bf *BlkFileV2) ReadFile(path string) (*BlkHeader, error)

func (*BlkFileV2) WithCompressor 

func (bf *BlkFileV2) WithCompressor(compressor compressor.Compressor) *BlkFileV2

func (*BlkFileV2) WriteBlock 

func (bf *BlkFileV2) WriteBlock(key string, reader io.Reader) (int64, error)

func (*BlkFileV2) WriteFile 

func (bf *BlkFileV2) WriteFile(path string) (int64, error)

type BlkHeader 

type BlkHeader struct {
	// block key(name)
	Key string

	// block size
	Size int64
}

func NewBlkHeader 

func NewBlkHeader(key string, size int64) *BlkHeader

func (*BlkHeader) Invalid 

func (h *BlkHeader) Invalid() bool

func (*BlkHeader) String 

func (h *BlkHeader) String() string

type BlkMeta 

type BlkMeta struct {
}

type BlockReadWriter 

type BlockReadWriter interface {
	io.Reader
	io.Writer
	io.Seeker
	io.Closer
	Sync() error
}

type BlocksV1Opt 

type BlocksV1Opt func(f *blockV1)

type BlocksV2Opt 

type BlocksV2Opt func(f *blockV2)

type FileHeader 

type FileHeader struct {
	MagicCode  uint32
	Version    uint16
	DataFormat uint16
	Reverse    uint16
}

func ReadFileHeader 

func ReadFileHeader(r io.Reader) (FileHeader, error)

type GetSizeFunc 

type GetSizeFunc func(string) int64

type JengaBlocks 

type JengaBlocks interface {
	Open(flag flags.OpenFlag) error
	Keys() []string
	WriteBlock(key string, reader io.Reader) (int64, error)
	ReadBlock(w io.Writer) (*BlkHeader, error)
	ReadBlockByKey(key string, writer io.Writer) (int64, error)
	Close() (err error)
	NeedSize() bool
	Flush() error
}

type KeyFilter 

type KeyFilter func(key string) bool

type KeySize 

type KeySize struct {
	Key  string
	Size int64
}

type Opener 

type Opener func(flag flags.OpenFlag) (rw BlockReadWriter, new bool, err error)

type VarInt 

type VarInt struct {
	// contains filtered or unexported fields
}

func NewFromReader 

func NewFromReader(r io.Reader) *VarInt

func (*VarInt) Bytes 

func (v *VarInt) Bytes() []byte

读取可变整数，完成返回true,未完成还需继续读取返回false

func (*VarInt) InitFromUInt64 

func (v *VarInt) InitFromUInt64(x uint64)

func (*VarInt) Length 

func (v *VarInt) Length() int

长度

func (*VarInt) Load 

func (v *VarInt) Load(d []byte) bool

读取可变整数，完成返回true,未完成还需继续读取返回false

func (*VarInt) LoadByte 

func (v *VarInt) LoadByte(d byte) bool

读取可变整数，完成返回true,未完成还需继续读取返回false

func (*VarInt) LoadFromReader 

func (v *VarInt) LoadFromReader(r io.Reader) (bool, int, error)

func (*VarInt) ToInt 

func (v *VarInt) ToInt() int64

func (*VarInt) ToUint 

func (v *VarInt) ToUint() uint64

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.