index

package
Version: v0.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 6, 2020 License: Apache-2.0 Imports: 24 Imported by: 10

Documentation

Index

Constants

View Source
const (
	ItemKindSnapshot = ".snp"
	ItemKindSegment  = ".seg"
)

Kinds of items managed by a Directory

View Source
const (
	EventKindCloseStart                 = 1 // when the index has started to close
	EventKindClose                      = 2 // when the index has been fully closed
	EventKindMergerProgress             = 3 // when the index has completed a round of merge operations
	EventKindPersisterProgress          = 4 // when the index has completed a round of persistence operations
	EventKindBatchIntroductionStart     = 5 // when the index has started to introduce a new batch
	EventKindBatchIntroduction          = 6 // when index has finished introducing a batch
	EventKindMergeTaskIntroductionStart = 7 // when the index has started to introduce a merge
	EventKindMergeTaskIntroduction      = 8 // when the index has finished introdocing a merge

)

Kinds of index events

Variables

This section is empty.

Functions

This section is empty.

Types

type Batch

type Batch struct {
	// contains filtered or unexported fields
}

func NewBatch

func NewBatch() *Batch

func (*Batch) Delete

func (b *Batch) Delete(id segment.Term)

func (*Batch) Insert

func (b *Batch) Insert(doc segment.Document)

func (*Batch) PersistedCallback

func (b *Batch) PersistedCallback() func(error)

func (*Batch) Reset

func (b *Batch) Reset()

func (*Batch) SetPersistedCallback

func (b *Batch) SetPersistedCallback(f func(error))

func (*Batch) Update

func (b *Batch) Update(id segment.Term, doc segment.Document)

type Config

type Config struct {
	SegmentType    string
	SegmentVersion uint32

	UnsafeBatch        bool
	EventCallback      func(Event)
	AsyncError         func(error)
	MergePlanOptions   mergeplan.Options
	NumAnalysisWorkers int
	AnalysisChan       chan func()
	GoFunc             func(func())
	DeletionPolicyFunc func() DeletionPolicy
	DirectoryFunc      func() Directory
	NormCalc           func(string, int) float32

	MergeBufferSize int

	// Optimizations
	OptimizeConjunction          bool
	OptimizeConjunctionUnadorned bool
	OptimizeDisjunctionUnadorned bool

	// MinSegmentsForInMemoryMerge represents the number of
	// in-memory zap segments that persistSnapshotMaybeMerge() needs to
	// see in an Snapshot before it decides to merge and persist
	// those segments
	MinSegmentsForInMemoryMerge int

	// PersisterNapTimeMSec controls the wait/delay injected into
	// persistence workloop to improve the chances for
	// a healthier and heavier in-memory merging
	PersisterNapTimeMSec int

	// PersisterNapTimeMSec > 0, and the number of files is less than
	// PersisterNapUnderNumFiles, then the persister will sleep
	// PersisterNapTimeMSec amount of time to improve the chances for
	// a healthier and heavier in-memory merging
	PersisterNapUnderNumFiles int

	// MemoryPressurePauseThreshold let persister to have a better leeway
	// for prudently performing the memory merge of segments on a memory
	// pressure situation. Here the config value is an upper threshold
	// for the number of paused application threads. The default value would
	// be a very high number to always favor the merging of memory segments.
	MemoryPressurePauseThreshold int

	ValidateSnapshotCRC bool
	// contains filtered or unexported fields
}

func DefaultConfig

func DefaultConfig(path string) Config

func DefaultConfigWithDirectory added in v0.1.3

func DefaultConfigWithDirectory(df func() Directory) Config

func InMemoryOnlyConfig

func InMemoryOnlyConfig() Config

func (Config) DisableOptimizeConjunction

func (config Config) DisableOptimizeConjunction() Config

func (Config) DisableOptimizeConjunctionUnadorned

func (config Config) DisableOptimizeConjunctionUnadorned() Config

func (Config) DisableOptimizeDisjunctionUnadorned

func (config Config) DisableOptimizeDisjunctionUnadorned() Config

func (Config) WithNormCalc

func (config Config) WithNormCalc(calc func(field string, numTerms int) float32) Config

func (Config) WithPersisterNapTimeMSec

func (config Config) WithPersisterNapTimeMSec(napTime int) Config

func (Config) WithSegmentPlugin

func (config Config) WithSegmentPlugin(plugin *SegmentPlugin) Config

func (Config) WithSegmentType

func (config Config) WithSegmentType(typ string) Config

func (Config) WithSegmentVersion

func (config Config) WithSegmentVersion(ver uint32) Config

func (Config) WithUnsafeBatches

func (config Config) WithUnsafeBatches() Config

func (Config) WithVirtualField

func (config Config) WithVirtualField(field segment.Field) Config

type DeletionPolicy

type DeletionPolicy interface {
	Commit(snapshot *Snapshot)
	Cleanup(Directory) error
}

type Directory

type Directory interface {

	// Setup is called first, allowing a directory to
	// perform additional set up, or return an error
	// indicating this directory cannot be used
	Setup(readOnly bool) error

	// List the ids of all the items of the specified kind
	// Items are returned in descending order by id
	List(kind string) ([]uint64, error)

	// Load the specified item
	// Item data is accessible via the returned *segment.Data structure
	// A io.Closer is returned, which must be called to release
	// resources held by this open item.
	Load(kind string, id uint64) (*segment.Data, io.Closer, error)

	// Persist a new item with data from the provided WriterTo
	// Implementations should monitor the closeCh and return with error
	// in the event it is closed before completion.
	Persist(kind string, id uint64, w WriterTo, closeCh chan struct{}) error

	// Remove the specified item
	Remove(kind string, id uint64) error

	// Stats returns total number of items and their cumulative size
	Stats() (numItems uint64, numBytes uint64)

	// Sync ensures directory metadata itself has been committed
	Sync() error

	// Lock ensures this process has exclusive access to write in this directory
	Lock() error

	// Unlock releases the lock held on this directory
	Unlock() error
}

Directory abstracts over a collection of items An item has a kind (string) and an id (uint64)

type Event

type Event struct {
	Kind     int
	Chill    *Writer
	Duration time.Duration
}

Event represents the information provided in an OnEvent() callback.

type FileSystemDirectory

type FileSystemDirectory struct {
	// contains filtered or unexported fields
}

func NewFileSystemDirectory

func NewFileSystemDirectory(path string) *FileSystemDirectory

func (*FileSystemDirectory) List

func (d *FileSystemDirectory) List(kind string) ([]uint64, error)

func (*FileSystemDirectory) Load

func (d *FileSystemDirectory) Load(kind string, id uint64) (*segment.Data, io.Closer, error)

func (*FileSystemDirectory) Lock

func (d *FileSystemDirectory) Lock() error

func (*FileSystemDirectory) Persist

func (d *FileSystemDirectory) Persist(kind string, id uint64, w WriterTo, closeCh chan struct{}) error

func (*FileSystemDirectory) Remove

func (d *FileSystemDirectory) Remove(kind string, id uint64) error

func (*FileSystemDirectory) Setup

func (d *FileSystemDirectory) Setup(readOnly bool) error

func (*FileSystemDirectory) Stats

func (d *FileSystemDirectory) Stats() (numFilesOnDisk, numBytesUsedDisk uint64)

func (*FileSystemDirectory) Sync

func (d *FileSystemDirectory) Sync() error

func (*FileSystemDirectory) Unlock

func (d *FileSystemDirectory) Unlock() error

type InMemoryDirectory

type InMemoryDirectory struct {
	// contains filtered or unexported fields
}

func NewInMemoryDirectory

func NewInMemoryDirectory() *InMemoryDirectory

func (*InMemoryDirectory) List

func (d *InMemoryDirectory) List(kind string) ([]uint64, error)

func (*InMemoryDirectory) Load

func (d *InMemoryDirectory) Load(kind string, id uint64) (*segment.Data, io.Closer, error)

func (*InMemoryDirectory) Lock

func (d *InMemoryDirectory) Lock() error

func (*InMemoryDirectory) Persist

func (d *InMemoryDirectory) Persist(kind string, id uint64, w WriterTo, closeCh chan struct{}) error

func (*InMemoryDirectory) Remove

func (d *InMemoryDirectory) Remove(kind string, id uint64) error

func (*InMemoryDirectory) Setup

func (d *InMemoryDirectory) Setup(readOnly bool) error

func (*InMemoryDirectory) Stats

func (d *InMemoryDirectory) Stats() (numItems, numBytes uint64)

func (*InMemoryDirectory) Sync

func (d *InMemoryDirectory) Sync() error

func (*InMemoryDirectory) Unlock

func (d *InMemoryDirectory) Unlock() error

type KeepNLatestDeletionPolicy

type KeepNLatestDeletionPolicy struct {
	// contains filtered or unexported fields
}

func NewKeepNLatestDeletionPolicy

func NewKeepNLatestDeletionPolicy(n int) *KeepNLatestDeletionPolicy

func (*KeepNLatestDeletionPolicy) Cleanup

func (p *KeepNLatestDeletionPolicy) Cleanup(dir Directory) error

func (*KeepNLatestDeletionPolicy) Commit

func (p *KeepNLatestDeletionPolicy) Commit(snapshot *Snapshot)

type SegmentPlugin

type SegmentPlugin struct {
	Type    string
	Version uint32
	New     func(results []segment.Document, normCalc func(string, int) float32) (segment.Segment, uint64, error)
	Load    func(*segment.Data) (segment.Segment, error)
	Merge   func([]segment.Segment, []*roaring.Bitmap, int) segment.Merger
}

type SegmentSnapshot

type SegmentSnapshot interface {
	ID() uint64
	Deleted() *roaring.Bitmap
}

type Snapshot

type Snapshot struct {
	// contains filtered or unexported fields
}

func OpenReader

func OpenReader(config Config) (*Snapshot, error)

func (*Snapshot) Backup

func (i *Snapshot) Backup(remote Directory, cancel chan struct{}) error

func (*Snapshot) Close

func (i *Snapshot) Close() error

func (*Snapshot) CollectionStats

func (i *Snapshot) CollectionStats(field string) (segment.CollectionStats, error)

func (*Snapshot) Count

func (i *Snapshot) Count() (uint64, error)

func (*Snapshot) DictionaryIterator

func (i *Snapshot) DictionaryIterator(field string, automaton segment.Automaton, start, end []byte) (
	segment.DictionaryIterator, error)

func (*Snapshot) DictionaryLookup

func (i *Snapshot) DictionaryLookup(field string) (segment.DictionaryLookup, error)

func (*Snapshot) DocumentValueReader

func (i *Snapshot) DocumentValueReader(fields []string) (
	segment.DocumentValueReader, error)

func (*Snapshot) Fields

func (i *Snapshot) Fields() ([]string, error)

func (*Snapshot) PostingsIterator

func (i *Snapshot) PostingsIterator(term []byte, field string, includeFreq,
	includeNorm, includeTermVectors bool) (segment.PostingsIterator, error)

func (*Snapshot) ReadFrom

func (i *Snapshot) ReadFrom(r io.Reader) (int64, error)

func (*Snapshot) Segments

func (i *Snapshot) Segments() []SegmentSnapshot

func (*Snapshot) Size

func (i *Snapshot) Size() int

func (*Snapshot) VisitStoredFields

func (i *Snapshot) VisitStoredFields(number uint64, visitor segment.StoredFieldVisitor) error

func (*Snapshot) WriteTo

func (i *Snapshot) WriteTo(w io.Writer, _ chan struct{}) (int64, error)

type Stats

type Stats struct {
	TotUpdates uint64
	TotDeletes uint64

	TotBatches        uint64
	TotBatchesEmpty   uint64
	TotBatchIntroTime uint64
	MaxBatchIntroTime uint64

	CurRootEpoch       uint64
	LastPersistedEpoch uint64
	LastMergedEpoch    uint64

	TotOnErrors uint64

	TotAnalysisTime uint64
	TotIndexTime    uint64

	TotIndexedPlainTextBytes uint64

	TotTermSearchersStarted  uint64
	TotTermSearchersFinished uint64

	TotIntroduceLoop       uint64
	TotIntroduceSegmentBeg uint64
	TotIntroduceSegmentEnd uint64
	TotIntroducePersistBeg uint64
	TotIntroducePersistEnd uint64
	TotIntroduceMergeBeg   uint64
	TotIntroduceMergeEnd   uint64
	TotIntroduceRevertBeg  uint64
	TotIntroduceRevertEnd  uint64

	TotIntroducedItems         uint64
	TotIntroducedSegmentsBatch uint64
	TotIntroducedSegmentsMerge uint64

	TotPersistLoopBeg          uint64
	TotPersistLoopErr          uint64
	TotPersistLoopProgress     uint64
	TotPersistLoopWait         uint64
	TotPersistLoopWaitNotified uint64
	TotPersistLoopEnd          uint64

	TotPersistedItems    uint64
	TotItemsToPersist    uint64
	TotPersistedSegments uint64

	TotPersisterSlowMergerPause  uint64
	TotPersisterSlowMergerResume uint64

	TotPersisterNapPauseCompleted uint64
	TotPersisterMergerNapBreak    uint64

	TotFileMergeLoopBeg uint64
	TotFileMergeLoopErr uint64
	TotFileMergeLoopEnd uint64

	TotFileMergePlan     uint64
	TotFileMergePlanErr  uint64
	TotFileMergePlanNone uint64
	TotFileMergePlanOk   uint64

	TotFileMergePlanTasks              uint64
	TotFileMergePlanTasksDone          uint64
	TotFileMergePlanTasksErr           uint64
	TotFileMergePlanTasksSegments      uint64
	TotFileMergePlanTasksSegmentsEmpty uint64

	TotFileMergeSegmentsEmpty uint64
	TotFileMergeSegments      uint64
	TotFileSegmentsAtRoot     uint64
	TotFileMergeWrittenBytes  uint64

	TotFileMergeZapBeg              uint64
	TotFileMergeZapEnd              uint64
	TotFileMergeZapTime             uint64
	MaxFileMergeZapTime             uint64
	TotFileMergeZapIntroductionTime uint64
	MaxFileMergeZapIntroductionTime uint64

	TotFileMergeIntroductions          uint64
	TotFileMergeIntroductionsDone      uint64
	TotFileMergeIntroductionsSkipped   uint64
	TotFileMergeIntroductionsObsoleted uint64

	CurFilesIneligibleForRemoval     uint64
	TotSnapshotsRemovedFromMetaStore uint64

	TotMemMergeBeg          uint64
	TotMemMergeErr          uint64
	TotMemMergeDone         uint64
	TotMemMergeZapBeg       uint64
	TotMemMergeZapEnd       uint64
	TotMemMergeZapTime      uint64
	MaxMemMergeZapTime      uint64
	TotMemMergeSegments     uint64
	TotMemorySegmentsAtRoot uint64

	TotEventFired    uint64
	TotEventReturned uint64

	CurOnDiskBytes           uint64
	CurOnDiskBytesUsedByRoot uint64 // FIXME not currently supported
	CurOnDiskFiles           uint64
	// contains filtered or unexported fields
}

Stats tracks statistics about the index, fields that are prefixed like CurXxxx are gauges (can go up and down), and fields that are prefixed like TotXxxx are monotonically increasing counters.

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func OpenWriter

func OpenWriter(config Config) (*Writer, error)

func (*Writer) Batch

func (s *Writer) Batch(batch *Batch) (err error)

Batch applies a batch of changes to the index atomically

func (*Writer) Close

func (s *Writer) Close() (err error)

func (*Writer) MemoryUsed

func (s *Writer) MemoryUsed() (memUsed uint64)

func (*Writer) Reader

func (s *Writer) Reader() (*Snapshot, error)

Reader returns a low-level accessor on the index data. Close it to release associated resources.

func (*Writer) Stats

func (s *Writer) Stats() Stats

type WriterOffline

type WriterOffline struct {
	// contains filtered or unexported fields
}

func OpenOfflineWriter

func OpenOfflineWriter(config Config) (writer *WriterOffline, err error)

func (*WriterOffline) Batch

func (s *WriterOffline) Batch(batch *Batch) (err error)

func (*WriterOffline) Close

func (s *WriterOffline) Close() error

type WriterTo

type WriterTo interface {
	WriteTo(w io.Writer, closeCh chan struct{}) (n int64, err error)
}

WriterTo is like io.WriterTo only it can be canceled by closing the closeCh

Directories

Path Synopsis
Package mergeplan provides a segment merge planning approach that's inspired by Lucene's TieredMergePolicy.java and descriptions like http://blog.mikemccandless.com/2011/02/visualizing-lucenes-segment-merges.html
Package mergeplan provides a segment merge planning approach that's inspired by Lucene's TieredMergePolicy.java and descriptions like http://blog.mikemccandless.com/2011/02/visualizing-lucenes-segment-merges.html

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL