storage

Documentation

Index

• Constants
• type Codec
• type DocBlock
  • func CompressDocBlock(src []byte, dst DocBlock, zstdLevel int) DocBlock
  • func NewBlock() DocBlock
  • func PackDocBlock(payload []byte, dst DocBlock) DocBlock
  • func (b DocBlock) CalcLen()
  • func (b DocBlock) Codec() Codec
  • func (b DocBlock) DecompressTo(dst []byte) ([]byte, error)
  • func (b DocBlock) FullLen() uint64
  • func (b DocBlock) GetExt1() uint64
  • func (b DocBlock) GetExt2() uint64
  • func (b DocBlock) Len() uint64
  • func (b DocBlock) Payload() []byte
  • func (b DocBlock) RawLen() uint64
  • func (b DocBlock) SetCodec(codecVal Codec)
  • func (b DocBlock) SetExt1(x uint64)
  • func (b DocBlock) SetExt2(x uint64)
  • func (b DocBlock) SetLen(val uint64)
  • func (b DocBlock) SetRawLen(x uint64)
• type DocBlocksReader
  • func NewDocBlocksReader(limiter *ReadLimiter, reader io.ReaderAt) DocBlocksReader
  • func (r *DocBlocksReader) ReadDocBlock(offset int64) ([]byte, uint64, error)
  • func (r *DocBlocksReader) ReadDocBlockPayload(offset int64) ([]byte, uint64, error)
• type DocsReader
  • func NewDocsReader(limiter *ReadLimiter, reader io.ReaderAt, docsCache *cache.Cache[[]byte]) DocsReader
  • func (r *DocsReader) ReadDocs(blockOffset uint64, docOffsets []uint64) ([][]byte, error)
  • func (r *DocsReader) ReadDocsFunc(blockOffset uint64, docOffsets []uint64, cb func([]byte) error) error
• type ImmutableFile
• type IndexBlockHeader
  • func NewEmptyIndexBlockHeader() IndexBlockHeader
  • func NewIndexBlockHeader(pos int64, ext1, ext2 uint64, size, rawSize uint32, codec Codec) IndexBlockHeader
  • func (b IndexBlockHeader) Codec() Codec
  • func (b IndexBlockHeader) GetExt1() uint64
  • func (b IndexBlockHeader) GetExt2() uint64
  • func (b IndexBlockHeader) GetPos() uint64
  • func (b IndexBlockHeader) Len() uint32
  • func (b IndexBlockHeader) RawLen() uint32
  • func (b IndexBlockHeader) SetCodec(codecVal Codec)
  • func (b IndexBlockHeader) SetExt1(x uint64)
  • func (b IndexBlockHeader) SetExt2(x uint64)
  • func (b IndexBlockHeader) SetLen(val uint32)
  • func (b IndexBlockHeader) SetPos(x uint64)
  • func (b IndexBlockHeader) SetRawLen(x uint32)
• type IndexReader
  • func NewIndexReader(limiter *ReadLimiter, readerName string, reader io.ReaderAt, ...) IndexReader
  • func (r *IndexReader) GetBlockHeader(index uint32) (IndexBlockHeader, error)
  • func (r *IndexReader) ReadIndexBlock(blockIndex uint32, dst []byte) ([]byte, uint64, error)
• type ReadLimiter
  • func NewReadLimiter(maxReadsNum int, counter prometheus.Counter) *ReadLimiter
  • func (rl *ReadLimiter) ReadAt(r io.ReaderAt, buf []byte, offset int64) (int, error)
• type Uploader

Constants

const (
	DocBlockHeaderLen = 33
)

const (
	IndexBlockHeaderSize = 33
)

Types

type Codec

type Codec byte

const (
	CodecNo Codec = iota
	CodecLZ4
	CodecZSTD
)

type DocBlock

type DocBlock []byte

func CompressDocBlock

func CompressDocBlock(src []byte, dst DocBlock, zstdLevel int) DocBlock

func NewBlock

func NewBlock() DocBlock

func PackDocBlock

func PackDocBlock(payload []byte, dst DocBlock) DocBlock

func (DocBlock) CalcLen

func (b DocBlock) CalcLen()

func (DocBlock) Codec

func (b DocBlock) Codec() Codec

func (DocBlock) DecompressTo

func (b DocBlock) DecompressTo(dst []byte) ([]byte, error)

DecompressTo always put the result in `dst` regardless of whether unpacking is required or part of the DocBlock can be enough.

So DocBlock does not share the same data with `dst` and can be used safely

func (DocBlock) FullLen

func (b DocBlock) FullLen() uint64

func (DocBlock) GetExt1

func (b DocBlock) GetExt1() uint64

func (DocBlock) GetExt2

func (b DocBlock) GetExt2() uint64

func (DocBlock) Len

func (b DocBlock) Len() uint64

func (DocBlock) Payload

func (b DocBlock) Payload() []byte

func (DocBlock) RawLen

func (b DocBlock) RawLen() uint64

func (DocBlock) SetCodec

func (b DocBlock) SetCodec(codecVal Codec)

func (DocBlock) SetExt1

func (b DocBlock) SetExt1(x uint64)

func (DocBlock) SetExt2

func (b DocBlock) SetExt2(x uint64)

func (DocBlock) SetLen

func (b DocBlock) SetLen(val uint64)

func (DocBlock) SetRawLen

func (b DocBlock) SetRawLen(x uint64)

type DocBlocksReader

type DocBlocksReader struct {
	// contains filtered or unexported fields
}

func NewDocBlocksReader

func NewDocBlocksReader(limiter *ReadLimiter, reader io.ReaderAt) DocBlocksReader

func (*DocBlocksReader) ReadDocBlock

func (r *DocBlocksReader) ReadDocBlock(offset int64) ([]byte, uint64, error)

func (*DocBlocksReader) ReadDocBlockPayload

func (r *DocBlocksReader) ReadDocBlockPayload(offset int64) ([]byte, uint64, error)

type DocsReader

type DocsReader struct {
	// contains filtered or unexported fields
}

func NewDocsReader

func NewDocsReader(limiter *ReadLimiter, reader io.ReaderAt, docsCache *cache.Cache[[]byte]) DocsReader

func (*DocsReader) ReadDocs

func (r *DocsReader) ReadDocs(blockOffset uint64, docOffsets []uint64) ([][]byte, error)

func (*DocsReader) ReadDocsFunc

func (r *DocsReader) ReadDocsFunc(blockOffset uint64, docOffsets []uint64, cb func([]byte) error) error

type ImmutableFile

type ImmutableFile interface {
	Name() string
	Stat() (os.FileInfo, error)

	io.Reader
	io.ReaderAt

	io.Seeker
	io.Closer
}

type IndexBlockHeader

type IndexBlockHeader []byte

func NewEmptyIndexBlockHeader

func NewEmptyIndexBlockHeader() IndexBlockHeader

func NewIndexBlockHeader

func NewIndexBlockHeader(pos int64, ext1, ext2 uint64, size, rawSize uint32, codec Codec) IndexBlockHeader

func (IndexBlockHeader) Codec

func (b IndexBlockHeader) Codec() Codec

func (IndexBlockHeader) GetExt1

func (b IndexBlockHeader) GetExt1() uint64

func (IndexBlockHeader) GetExt2

func (b IndexBlockHeader) GetExt2() uint64

func (IndexBlockHeader) GetPos

func (b IndexBlockHeader) GetPos() uint64

func (IndexBlockHeader) Len

func (b IndexBlockHeader) Len() uint32

func (IndexBlockHeader) RawLen

func (b IndexBlockHeader) RawLen() uint32

func (IndexBlockHeader) SetCodec

func (b IndexBlockHeader) SetCodec(codecVal Codec)

func (IndexBlockHeader) SetExt1

func (b IndexBlockHeader) SetExt1(x uint64)

func (IndexBlockHeader) SetExt2

func (b IndexBlockHeader) SetExt2(x uint64)

func (IndexBlockHeader) SetLen

func (b IndexBlockHeader) SetLen(val uint32)

func (IndexBlockHeader) SetPos

func (b IndexBlockHeader) SetPos(x uint64)

func (IndexBlockHeader) SetRawLen

func (b IndexBlockHeader) SetRawLen(x uint32)

type IndexReader

type IndexReader struct {
	// contains filtered or unexported fields
}

func NewIndexReader

func NewIndexReader(
	limiter *ReadLimiter, readerName string,
	reader io.ReaderAt, registryCache *cache.Cache[[]byte],
) IndexReader

func (*IndexReader) GetBlockHeader

func (r *IndexReader) GetBlockHeader(index uint32) (IndexBlockHeader, error)

func (*IndexReader) ReadIndexBlock

func (r *IndexReader) ReadIndexBlock(blockIndex uint32, dst []byte) ([]byte, uint64, error)

type ReadLimiter

type ReadLimiter struct {
	// contains filtered or unexported fields
}

func NewReadLimiter

func NewReadLimiter(maxReadsNum int, counter prometheus.Counter) *ReadLimiter

func (*ReadLimiter) ReadAt

func (rl *ReadLimiter) ReadAt(r io.ReaderAt, buf []byte, offset int64) (int, error)

type Uploader

type Uploader interface {
	Upload(context.Context, ImmutableFile) error
}