archiver

Documentation

Index

• func GenerateBlockID(timestampStart string, timestampEnd string) string
• type Archiver
  • func New(opts *ArchiverOptions, logger logging.LoggerContract) Archiver
• type ArchiverImpl
  • func (a *ArchiverImpl) Archive(ctx context.Context, streamName string, messages []rdb.XMessage) error
  • func (a *ArchiverImpl) CreateBloomFilter(messages []rdb.XMessage, meta *block.BlockMetadata) (io.ReadCloser, error)
  • func (a *ArchiverImpl) CreateMetadata(meta *block.BlockMetadata) ([]byte, error)
  • func (a *ArchiverImpl) SerializeToParquet(messages []rdb.XMessage, meta *block.BlockMetadata) (io.ReadCloser, error)
• type ArchiverOptions
• type BufferFile
  • func NewBufferFile(buffer *bytes.Buffer, offset int64) *BufferFile
  • func (f *BufferFile) Close() error
  • func (f *BufferFile) Create(name string) (source.ParquetFile, error)
  • func (f *BufferFile) Open(name string) (source.ParquetFile, error)
  • func (f *BufferFile) Read(p []byte) (int, error)
  • func (f *BufferFile) Seek(offset int64, whence int) (int64, error)
  • func (f *BufferFile) Write(p []byte) (int, error)

Functions

func GenerateBlockID

func GenerateBlockID(timestampStart string, timestampEnd string) string

Generates a unique block ID based on the start and end timestamps

Types

type Archiver

type Archiver interface {
	Archive(ctx context.Context, streamName string, messages []rdb.XMessage) error
}

func New

func New(opts *ArchiverOptions, logger logging.LoggerContract) Archiver

Create a new Archiver instance

type ArchiverImpl

type ArchiverImpl struct {
	Storage storage.Storage
	Logger  logging.LoggerContract
}

func (*ArchiverImpl) Archive

func (a *ArchiverImpl) Archive(ctx context.Context, streamName string, messages []rdb.XMessage) error

func (*ArchiverImpl) CreateBloomFilter

func (a *ArchiverImpl) CreateBloomFilter(messages []rdb.XMessage, meta *block.BlockMetadata) (io.ReadCloser, error)

CreateBloomFilter generates a bloom filter containing all message IDs in the block

func (*ArchiverImpl) CreateMetadata

func (a *ArchiverImpl) CreateMetadata(meta *block.BlockMetadata) ([]byte, error)

func (*ArchiverImpl) SerializeToParquet

func (a *ArchiverImpl) SerializeToParquet(messages []rdb.XMessage, meta *block.BlockMetadata) (io.ReadCloser, error)

type ArchiverOptions

type ArchiverOptions struct {
	Storage storage.Storage
}

type BufferFile

type BufferFile struct {
	Buffer     *bytes.Buffer
	ReadBuffer *bytes.Reader
	Offset     int64
}

BufferFile is an implementation of the ParquetFile interface that writes to an in-memory buffer

func NewBufferFile

func NewBufferFile(buffer *bytes.Buffer, offset int64) *BufferFile

func (*BufferFile) Close

func (f *BufferFile) Close() error

func (*BufferFile) Create

func (f *BufferFile) Create(name string) (source.ParquetFile, error)

func (*BufferFile) Open

func (f *BufferFile) Open(name string) (source.ParquetFile, error)

func (*BufferFile) Read

func (f *BufferFile) Read(p []byte) (int, error)

func (*BufferFile) Seek

func (f *BufferFile) Seek(offset int64, whence int) (int64, error)

func (*BufferFile) Write

func (f *BufferFile) Write(p []byte) (int, error)