firestorm

package
v0.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 16, 2016 License: Apache-2.0 Imports: 21 Imported by: 0

Documentation

Overview

Package firestorm is a generated protocol buffer package.

It is generated from these files:

firestorm_rows.proto

It has these top-level messages:

VersionValue
FieldValue
DictionaryValue
TermVector
TermFreqValue
StoredValue

Index

Constants

View Source
const ByteSeparator byte = 0xff
View Source
const DefaultDictUpdateThreshold = 10
View Source
const DefaultGarbageThreshold = 10
View Source
const DefaultMaxDocsPerPass = 1000
View Source
const IDFieldName = "_id"
View Source
const Name = "firestorm"
View Source
const Version uint64 = 1

Variables

View Source
var DefaultDictUpdateSleep = 1 * time.Second
View Source
var DefaultGarbageSleep = 15 * time.Second
View Source
var DictionaryKeyPrefix = []byte{'d'}
View Source
var (
	ErrInvalidLengthFirestormRows = fmt.Errorf("proto: negative length found during unmarshaling")
)
View Source
var FieldKeyPrefix = []byte{'f'}
View Source
var IncompatibleVersion = fmt.Errorf("incompatible version, %d is supported", Version)
View Source
var InternalKeyPrefix = []byte{'i'}
View Source
var StoredKeyPrefix = []byte{'s'}
View Source
var TermFreqKeyPrefix = []byte{'t'}
View Source
var VersionKey = []byte{'v'}

Functions

func DictionaryRowKey

func DictionaryRowKey(field uint16, term []byte) []byte

func NewFirestorm

func NewFirestorm(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error)

func SetLog

func SetLog(l *log.Logger)

SetLog sets the logger used for logging by default log messages are sent to ioutil.Discard

func StoredIteratorStartDocID

func StoredIteratorStartDocID(docID []byte) []byte

func StoredPrefixDocIDNum

func StoredPrefixDocIDNum(docID []byte, docNum uint64) []byte

func TermFreqIteratorStart

func TermFreqIteratorStart(field uint16, term []byte) []byte

func TermFreqPrefixFieldTermDocId

func TermFreqPrefixFieldTermDocId(field uint16, term []byte, docID []byte) []byte

Types

type Compensator

type Compensator struct {
	// contains filtered or unexported fields
}

func NewCompensator

func NewCompensator() *Compensator

func (*Compensator) GarbageCollect

func (c *Compensator) GarbageCollect(docNums []uint64)

func (*Compensator) GarbageCount

func (c *Compensator) GarbageCount() uint64

func (*Compensator) Migrate

func (c *Compensator) Migrate(docID []byte, docNum uint64, oldDocNums []uint64)

func (*Compensator) Mutate

func (c *Compensator) Mutate(docID []byte, docNum uint64)

func (*Compensator) MutateBatch

func (c *Compensator) MutateBatch(inflightItems []*InFlightItem, lastDocNum uint64)

func (*Compensator) Snapshot

func (c *Compensator) Snapshot() *Snapshot

type DictUpdater

type DictUpdater struct {
	// contains filtered or unexported fields
}

func NewDictUpdater

func NewDictUpdater(f *Firestorm) *DictUpdater

func (*DictUpdater) Notify

func (d *DictUpdater) Notify(term string, usage int64)

func (*DictUpdater) NotifyBatch

func (d *DictUpdater) NotifyBatch(termUsages map[string]int64)

func (*DictUpdater) Start

func (d *DictUpdater) Start()

func (*DictUpdater) Stop

func (d *DictUpdater) Stop()

type DictionaryRow

type DictionaryRow struct {
	// contains filtered or unexported fields
}

func NewDictionaryRow

func NewDictionaryRow(field uint16, term []byte, count uint64) *DictionaryRow

func NewDictionaryRowK

func NewDictionaryRowK(key []byte) (*DictionaryRow, error)

func NewDictionaryRowKV

func NewDictionaryRowKV(key, value []byte) (*DictionaryRow, error)

func (*DictionaryRow) Count

func (dr *DictionaryRow) Count() uint64

func (*DictionaryRow) Key

func (dr *DictionaryRow) Key() []byte

func (*DictionaryRow) KeySize

func (dr *DictionaryRow) KeySize() int

func (*DictionaryRow) KeyTo

func (dr *DictionaryRow) KeyTo(buf []byte) (int, error)

func (*DictionaryRow) SetCount

func (dr *DictionaryRow) SetCount(count uint64)

func (*DictionaryRow) Value

func (dr *DictionaryRow) Value() []byte

func (*DictionaryRow) ValueSize

func (dr *DictionaryRow) ValueSize() int

func (*DictionaryRow) ValueTo

func (dr *DictionaryRow) ValueTo(buf []byte) (int, error)

type DictionaryValue

type DictionaryValue struct {
	Count            *uint64 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (*DictionaryValue) GetCount

func (m *DictionaryValue) GetCount() uint64

func (*DictionaryValue) Marshal

func (m *DictionaryValue) Marshal() (data []byte, err error)

func (*DictionaryValue) MarshalTo

func (m *DictionaryValue) MarshalTo(data []byte) (n int, err error)

func (*DictionaryValue) ProtoMessage

func (*DictionaryValue) ProtoMessage()

func (*DictionaryValue) Reset

func (m *DictionaryValue) Reset()

func (*DictionaryValue) Size

func (m *DictionaryValue) Size() (n int)

func (*DictionaryValue) String

func (m *DictionaryValue) String() string

func (*DictionaryValue) Unmarshal

func (m *DictionaryValue) Unmarshal(data []byte) error

type DocNumberList

type DocNumberList []uint64

func (DocNumberList) HighestValid

func (l DocNumberList) HighestValid(maxRead uint64) uint64

HighestValid returns the highest valid doc number from a *SORTED* DocNumberList if no doc number in the list is valid, then 0

func (DocNumberList) Len

func (l DocNumberList) Len() int

func (DocNumberList) Less

func (l DocNumberList) Less(i, j int) bool

func (DocNumberList) Swap

func (l DocNumberList) Swap(i, j int)

type FieldRow

type FieldRow struct {
	// contains filtered or unexported fields
}

func NewFieldRow

func NewFieldRow(i uint16, name string) *FieldRow

func NewFieldRowKV

func NewFieldRowKV(key, value []byte) (*FieldRow, error)

func (*FieldRow) Index

func (fr *FieldRow) Index() uint16

func (*FieldRow) Key

func (fr *FieldRow) Key() []byte

func (*FieldRow) KeySize

func (fr *FieldRow) KeySize() int

func (*FieldRow) KeyTo

func (fr *FieldRow) KeyTo(buf []byte) (int, error)

func (*FieldRow) Name

func (fr *FieldRow) Name() string

func (*FieldRow) String

func (fr *FieldRow) String() string

func (*FieldRow) Value

func (fr *FieldRow) Value() []byte

func (*FieldRow) ValueSize

func (fr *FieldRow) ValueSize() int

func (*FieldRow) ValueTo

func (fr *FieldRow) ValueTo(buf []byte) (int, error)

type FieldValue

type FieldValue struct {
	Name             *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (*FieldValue) GetName

func (m *FieldValue) GetName() string

func (*FieldValue) Marshal

func (m *FieldValue) Marshal() (data []byte, err error)

func (*FieldValue) MarshalTo

func (m *FieldValue) MarshalTo(data []byte) (n int, err error)

func (*FieldValue) ProtoMessage

func (*FieldValue) ProtoMessage()

func (*FieldValue) Reset

func (m *FieldValue) Reset()

func (*FieldValue) Size

func (m *FieldValue) Size() (n int)

func (*FieldValue) String

func (m *FieldValue) String() string

func (*FieldValue) Unmarshal

func (m *FieldValue) Unmarshal(data []byte) error

type Firestorm

type Firestorm struct {
	// contains filtered or unexported fields
}

func (*Firestorm) Advanced

func (f *Firestorm) Advanced() (store.KVStore, error)

func (*Firestorm) Analyze

func (*Firestorm) Batch

func (f *Firestorm) Batch(batch *index.Batch) (err error)

func (*Firestorm) Close

func (f *Firestorm) Close() error

func (*Firestorm) Delete

func (f *Firestorm) Delete(id string) error

func (*Firestorm) DeleteInternal

func (f *Firestorm) DeleteInternal(key []byte) (err error)

func (*Firestorm) DocCount

func (f *Firestorm) DocCount() (uint64, error)

func (*Firestorm) DumpAll

func (f *Firestorm) DumpAll() chan interface{}

func (*Firestorm) DumpDoc

func (f *Firestorm) DumpDoc(docID string) chan interface{}

func (*Firestorm) DumpFields

func (f *Firestorm) DumpFields() chan interface{}

func (*Firestorm) Open

func (f *Firestorm) Open() (err error)

func (*Firestorm) Reader

func (f *Firestorm) Reader() (index.IndexReader, error)

func (*Firestorm) SetInternal

func (f *Firestorm) SetInternal(key, val []byte) (err error)

func (*Firestorm) Stats

func (f *Firestorm) Stats() json.Marshaler

func (*Firestorm) StatsMap

func (f *Firestorm) StatsMap() map[string]interface{}

func (*Firestorm) Update

func (f *Firestorm) Update(doc *document.Document) (err error)

func (*Firestorm) Wait

func (f *Firestorm) Wait(timeout time.Duration) error

type GarbageCollector

type GarbageCollector struct {
	// contains filtered or unexported fields
}

func NewGarbageCollector

func NewGarbageCollector(f *Firestorm) *GarbageCollector

func (*GarbageCollector) NextBatch

func (gc *GarbageCollector) NextBatch(n int) []uint64

func (*GarbageCollector) Notify

func (gc *GarbageCollector) Notify(docNum uint64, docId []byte)

func (*GarbageCollector) Start

func (gc *GarbageCollector) Start()

func (*GarbageCollector) Stop

func (gc *GarbageCollector) Stop()

type InFlightItem

type InFlightItem struct {
	// contains filtered or unexported fields
}

type InternalRow

type InternalRow struct {
	// contains filtered or unexported fields
}

func NewInternalRow

func NewInternalRow(key, val []byte) *InternalRow

func NewInternalRowKV

func NewInternalRowKV(key, value []byte) (*InternalRow, error)

func (*InternalRow) Key

func (ir *InternalRow) Key() []byte

func (*InternalRow) KeySize

func (ir *InternalRow) KeySize() int

func (*InternalRow) KeyTo

func (ir *InternalRow) KeyTo(buf []byte) (int, error)

func (*InternalRow) String

func (ir *InternalRow) String() string

func (*InternalRow) Value

func (ir *InternalRow) Value() []byte

func (*InternalRow) ValueSize

func (ir *InternalRow) ValueSize() int

func (*InternalRow) ValueTo

func (ir *InternalRow) ValueTo(buf []byte) (int, error)

type KVVisitor

type KVVisitor func(key, val []byte) (bool, error)

type Lookuper

type Lookuper struct {
	// contains filtered or unexported fields
}

func NewLookuper

func NewLookuper(f *Firestorm) *Lookuper

func (*Lookuper) NotifyBatch

func (l *Lookuper) NotifyBatch(items []*InFlightItem)

func (*Lookuper) Start

func (l *Lookuper) Start()

func (*Lookuper) Stop

func (l *Lookuper) Stop()

type Snapshot

type Snapshot struct {
	// contains filtered or unexported fields
}

func (*Snapshot) Valid

func (s *Snapshot) Valid(docID []byte, docNum uint64) bool

func (*Snapshot) Which

func (s *Snapshot) Which(docID []byte, docNumList DocNumberList) uint64

returns which doc number is valid if none, then 0

type StoredRow

type StoredRow struct {
	// contains filtered or unexported fields
}

func NewStoredRow

func NewStoredRow(docID []byte, docNum uint64, field uint16, arrayPositions []uint64, value []byte) *StoredRow

func NewStoredRowKV

func NewStoredRowKV(key, value []byte) (*StoredRow, error)

func (*StoredRow) DocID

func (sr *StoredRow) DocID() []byte

func (*StoredRow) DocNum

func (sr *StoredRow) DocNum() uint64

func (*StoredRow) Key

func (sr *StoredRow) Key() []byte

func (*StoredRow) KeySize

func (sr *StoredRow) KeySize() int

func (*StoredRow) KeyTo

func (sr *StoredRow) KeyTo(buf []byte) (int, error)

func (*StoredRow) ParseKey

func (sr *StoredRow) ParseKey(key []byte) error

func (*StoredRow) String

func (sr *StoredRow) String() string

func (*StoredRow) Value

func (sr *StoredRow) Value() []byte

func (*StoredRow) ValueSize

func (sr *StoredRow) ValueSize() int

func (*StoredRow) ValueTo

func (sr *StoredRow) ValueTo(buf []byte) (int, error)

type StoredValue

type StoredValue struct {
	Raw              []byte `protobuf:"bytes,1,opt,name=raw" json:"raw,omitempty"`
	XXX_unrecognized []byte `json:"-"`
}

func (*StoredValue) GetRaw

func (m *StoredValue) GetRaw() []byte

func (*StoredValue) Marshal

func (m *StoredValue) Marshal() (data []byte, err error)

func (*StoredValue) MarshalTo

func (m *StoredValue) MarshalTo(data []byte) (n int, err error)

func (*StoredValue) ProtoMessage

func (*StoredValue) ProtoMessage()

func (*StoredValue) Reset

func (m *StoredValue) Reset()

func (*StoredValue) Size

func (m *StoredValue) Size() (n int)

func (*StoredValue) String

func (m *StoredValue) String() string

func (*StoredValue) Unmarshal

func (m *StoredValue) Unmarshal(data []byte) error

type TermFreqRow

type TermFreqRow struct {
	// contains filtered or unexported fields
}

func InitTermFreqRow

func InitTermFreqRow(tfr *TermFreqRow, field uint16, term []byte, docID []byte, docNum uint64, freq uint64, norm float32, termVectors []*TermVector) *TermFreqRow

func NewTermFreqRow

func NewTermFreqRow(field uint16, term []byte, docID []byte, docNum uint64, freq uint64, norm float32, termVectors []*TermVector) *TermFreqRow

func NewTermFreqRowKV

func NewTermFreqRowKV(key, value []byte) (*TermFreqRow, error)

func (*TermFreqRow) DictionaryRowKey

func (tfr *TermFreqRow) DictionaryRowKey() []byte

func (*TermFreqRow) DictionaryRowKeySize

func (tfr *TermFreqRow) DictionaryRowKeySize() int

func (*TermFreqRow) DictionaryRowKeyTo

func (tfr *TermFreqRow) DictionaryRowKeyTo(buf []byte) (int, error)

func (*TermFreqRow) DocID

func (tfr *TermFreqRow) DocID() []byte

func (*TermFreqRow) DocNum

func (tfr *TermFreqRow) DocNum() uint64

func (*TermFreqRow) Field

func (tfr *TermFreqRow) Field() uint16

func (*TermFreqRow) Freq

func (tfr *TermFreqRow) Freq() uint64

func (*TermFreqRow) Key

func (tfr *TermFreqRow) Key() []byte

func (*TermFreqRow) KeySize

func (tfr *TermFreqRow) KeySize() int

func (*TermFreqRow) KeyTo

func (tfr *TermFreqRow) KeyTo(buf []byte) (int, error)

func (*TermFreqRow) Norm

func (tfr *TermFreqRow) Norm() float32

func (*TermFreqRow) ParseKey

func (tfr *TermFreqRow) ParseKey(key []byte) error

func (*TermFreqRow) String

func (tfr *TermFreqRow) String() string

func (*TermFreqRow) Term

func (tfr *TermFreqRow) Term() []byte

func (*TermFreqRow) Value

func (tfr *TermFreqRow) Value() []byte

func (*TermFreqRow) ValueSize

func (tfr *TermFreqRow) ValueSize() int

func (*TermFreqRow) ValueTo

func (tfr *TermFreqRow) ValueTo(buf []byte) (int, error)

func (*TermFreqRow) Vectors

func (tfr *TermFreqRow) Vectors() []*TermVector

type TermFreqValue

type TermFreqValue struct {
	Freq             *uint64       `protobuf:"varint,1,req,name=freq" json:"freq,omitempty"`
	Norm             *float32      `protobuf:"fixed32,2,opt,name=norm" json:"norm,omitempty"`
	Vectors          []*TermVector `protobuf:"bytes,3,rep,name=vectors" json:"vectors,omitempty"`
	XXX_unrecognized []byte        `json:"-"`
}

func (*TermFreqValue) GetFreq

func (m *TermFreqValue) GetFreq() uint64

func (*TermFreqValue) GetNorm

func (m *TermFreqValue) GetNorm() float32

func (*TermFreqValue) GetVectors

func (m *TermFreqValue) GetVectors() []*TermVector

func (*TermFreqValue) Marshal

func (m *TermFreqValue) Marshal() (data []byte, err error)

func (*TermFreqValue) MarshalTo

func (m *TermFreqValue) MarshalTo(data []byte) (n int, err error)

func (*TermFreqValue) ProtoMessage

func (*TermFreqValue) ProtoMessage()

func (*TermFreqValue) Reset

func (m *TermFreqValue) Reset()

func (*TermFreqValue) Size

func (m *TermFreqValue) Size() (n int)

func (*TermFreqValue) String

func (m *TermFreqValue) String() string

func (*TermFreqValue) Unmarshal

func (m *TermFreqValue) Unmarshal(data []byte) error

type TermVector

type TermVector struct {
	Field            *uint32  `protobuf:"varint,1,opt,name=field" json:"field,omitempty"`
	Pos              *uint64  `protobuf:"varint,2,opt,name=pos" json:"pos,omitempty"`
	Start            *uint64  `protobuf:"varint,3,opt,name=start" json:"start,omitempty"`
	End              *uint64  `protobuf:"varint,4,opt,name=end" json:"end,omitempty"`
	ArrayPositions   []uint64 `protobuf:"varint,5,rep,name=arrayPositions" json:"arrayPositions,omitempty"`
	XXX_unrecognized []byte   `json:"-"`
}

func NewTermVector

func NewTermVector(field uint16, pos uint64, start uint64, end uint64, arrayPos []uint64) *TermVector

func (*TermVector) GetArrayPositions

func (m *TermVector) GetArrayPositions() []uint64

func (*TermVector) GetEnd

func (m *TermVector) GetEnd() uint64

func (*TermVector) GetField

func (m *TermVector) GetField() uint32

func (*TermVector) GetPos

func (m *TermVector) GetPos() uint64

func (*TermVector) GetStart

func (m *TermVector) GetStart() uint64

func (*TermVector) Marshal

func (m *TermVector) Marshal() (data []byte, err error)

func (*TermVector) MarshalTo

func (m *TermVector) MarshalTo(data []byte) (n int, err error)

func (*TermVector) ProtoMessage

func (*TermVector) ProtoMessage()

func (*TermVector) Reset

func (m *TermVector) Reset()

func (*TermVector) Size

func (m *TermVector) Size() (n int)

func (*TermVector) String

func (m *TermVector) String() string

func (*TermVector) Unmarshal

func (m *TermVector) Unmarshal(data []byte) error

type VersionRow

type VersionRow struct {
	// contains filtered or unexported fields
}

func NewVersionRow

func NewVersionRow(version uint64) *VersionRow

func NewVersionRowV

func NewVersionRowV(val []byte) (*VersionRow, error)

func (*VersionRow) Key

func (vr *VersionRow) Key() []byte

func (*VersionRow) KeySize

func (vr *VersionRow) KeySize() int

func (*VersionRow) KeyTo

func (vr *VersionRow) KeyTo(buf []byte) (int, error)

func (*VersionRow) Value

func (vr *VersionRow) Value() []byte

func (*VersionRow) ValueSize

func (vr *VersionRow) ValueSize() int

func (*VersionRow) ValueTo

func (vr *VersionRow) ValueTo(buf []byte) (int, error)

func (*VersionRow) Version

func (vr *VersionRow) Version() uint64

type VersionValue

type VersionValue struct {
	Version          *uint64 `protobuf:"varint,1,req,name=version" json:"version,omitempty"`
	XXX_unrecognized []byte  `json:"-"`
}

func (*VersionValue) GetVersion

func (m *VersionValue) GetVersion() uint64

func (*VersionValue) Marshal

func (m *VersionValue) Marshal() (data []byte, err error)

func (*VersionValue) MarshalTo

func (m *VersionValue) MarshalTo(data []byte) (n int, err error)

func (*VersionValue) ProtoMessage

func (*VersionValue) ProtoMessage()

func (*VersionValue) Reset

func (m *VersionValue) Reset()

func (*VersionValue) Size

func (m *VersionValue) Size() (n int)

func (*VersionValue) String

func (m *VersionValue) String() string

func (*VersionValue) Unmarshal

func (m *VersionValue) Unmarshal(data []byte) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL