segment

README

Scorch Segment API

Scorch supports a pluggable Segment interface.

By placing these interfaces in their own, hopefully slowly evolving module, it frees up Scorch and the underlying segment to each introduce new major versions without interfering with one another.

With that in mind, we anticipate introducing non-breaking changes only to this module, and keeping the major version at 1.x for some time.

Documentation

Index

• Variables
• type Automaton
• type DictionaryIterator
• type DiskStatsReporter
• type DocValueVisitable
• type DocVisitState
• type FieldStats
• type FieldStatsReporter
• type Location
• type OptimizablePostingsIterator
• type PersistedSegment
• type Posting
• type PostingsIterator
• type PostingsList
• type Segment
• type StatsReporter
• type StoredFieldValueVisitor
• type Synonym
• type SynonymsIterator
• type SynonymsList
• type TermDictionary
• type Thesaurus
• type ThesaurusIterator
• type ThesaurusSegment
• type UnpersistedSegment

Variables

var ErrClosed = fmt.Errorf("index closed")

Types

type Automaton

type Automaton interface {

	// Start returns the start state
	Start() int

	// IsMatch returns true if and only if the state is a match
	IsMatch(int) bool

	// CanMatch returns true if and only if it is possible to reach a match
	// in zero or more steps
	CanMatch(int) bool

	// WillAlwaysMatch returns true if and only if the current state matches
	// and will always match no matter what steps are taken
	WillAlwaysMatch(int) bool

	// Accept returns the next state given the input to the specified state
	Accept(int, byte) int
}

Automaton represents the general contract of a byte-based finite automaton

type DictionaryIterator

type DictionaryIterator interface {
	Next() (*index.DictEntry, error)
}

type DiskStatsReporter

type DiskStatsReporter interface {
	// BytesRead returns the bytes read from the disk as
	// part of the current running query.
	BytesRead() uint64

	// ResetBytesRead is used by the parent layer
	// to reset the bytes read value to a consistent
	// value during operations such as merging of segments.
	ResetBytesRead(uint64)

	// BytesWritten returns the bytes written to disk while
	// building an index
	BytesWritten() uint64
}

type DocValueVisitable

type DocValueVisitable interface {
	VisitDocValues(localDocNum uint64, fields []string,
		visitor index.DocValueVisitor, optional DocVisitState) (DocVisitState, error)

	// VisitableDocValueFields implementation should return
	// the list of fields which are document value persisted and
	// therefore visitable by the above VisitDocValues method.
	VisitableDocValueFields() ([]string, error)
}

DocValueVisitable is implemented by various scorch segment implementations with persistence for the un inverting of the postings or other indexed values.

type DocVisitState

type DocVisitState interface {
	DiskStatsReporter
}

type FieldStats

type FieldStats interface {
	Store(statName, fieldName string, value uint64)
	Aggregate(stats FieldStats)
	Fetch() map[string]map[string]uint64
}

type FieldStatsReporter

type FieldStatsReporter interface {
	UpdateFieldStats(FieldStats)
}

type Location

type Location interface {
	Field() string
	Start() uint64
	End() uint64
	Pos() uint64
	ArrayPositions() []uint64
	Size() int
}

type OptimizablePostingsIterator

type OptimizablePostingsIterator interface {
	ActualBitmap() *roaring.Bitmap
	DocNum1Hit() (uint64, bool)
	ReplaceActual(*roaring.Bitmap)
}

type PersistedSegment

type PersistedSegment interface {
	Segment
	Path() string
}

type Posting

type Posting interface {
	Number() uint64

	Frequency() uint64
	Norm() float64

	Locations() []Location

	Size() int
}

type PostingsIterator

type PostingsIterator interface {
	DiskStatsReporter

	// The caller is responsible for copying whatever it needs from
	// the returned Posting instance before calling Next(), as some
	// implementations may return a shared instance to reduce memory
	// allocations.
	Next() (Posting, error)

	// Advance will return the posting with the specified doc number
	// or if there is no such posting, the next posting.
	// Callers MUST NOT attempt to pass a docNum that is less than or
	// equal to the currently visited posting doc Num.
	Advance(docNum uint64) (Posting, error)

	Size() int
}

type PostingsList

type PostingsList interface {
	DiskStatsReporter

	Iterator(includeFreq, includeNorm, includeLocations bool, prealloc PostingsIterator) PostingsIterator

	Size() int

	Count() uint64
}

type Segment

type Segment interface {
	DiskStatsReporter

	Dictionary(field string) (TermDictionary, error)

	VisitStoredFields(num uint64, visitor StoredFieldValueVisitor) error

	DocID(num uint64) ([]byte, error)

	Count() uint64

	DocNumbers([]string) (*roaring.Bitmap, error)

	Fields() []string

	Close() error

	Size() int

	AddRef()
	DecRef() error
}

type StatsReporter

type StatsReporter interface {
	ReportBytesWritten(bytesWritten uint64)
}

type StoredFieldValueVisitor

type StoredFieldValueVisitor func(field string, typ byte, value []byte, pos []uint64) bool

StoredFieldValueVisitor defines a callback to be visited for each stored field value. The return value determines if the visitor should keep going. Returning true continues visiting, false stops.

type Synonym

type Synonym interface {
	// Number returns the document number from which the synonym originates.
	Number() uint32
	// Term returns the textual representation of the synonym.
	Term() string

	Size() int
}

Synonym represents a single synonym for a term in the thesaurus.

type SynonymsIterator

type SynonymsIterator interface {
	// Next returns the next synonym in the list or an error if iteration fails.
	Next() (Synonym, error)

	Size() int
}

SynonymsIterator provides a mechanism to iterate over a list of synonyms.

type SynonymsList

type SynonymsList interface {
	// Iterator returns an iterator to traverse the list of synonyms.
	// The `prealloc` parameter can be used to reuse existing memory for the iterator.
	Iterator(prealloc SynonymsIterator) SynonymsIterator

	Size() int
}

SynonymsList represents a list of synonyms for a term.

type TermDictionary

type TermDictionary interface {
	PostingsList(term []byte, except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error)

	AutomatonIterator(a Automaton,
		startKeyInclusive, endKeyExclusive []byte) DictionaryIterator

	Contains(key []byte) (bool, error)

	// returns total number of terms in the term dictionary
	Cardinality() int
}

type Thesaurus

type Thesaurus interface {
	// SynonymsList retrieves a list of synonyms for the specified term. The `except` parameter
	// excludes specific synonyms, such as those originating from deleted documents. The `prealloc`
	// parameter allows the use of preallocated memory to optimize performance.
	SynonymsList(term []byte, except *roaring.Bitmap, prealloc SynonymsList) (SynonymsList, error)

	// AutomatonIterator creates an iterator over the thesaurus keys/terms using the provided automaton.
	// The iteration is constrained by the specified key range [startKeyInclusive, endKeyExclusive).
	// These terms or keys are the ones that have a SynonymsList associated with them, in the thesaurus.
	AutomatonIterator(a Automaton, startKeyInclusive, endKeyExclusive []byte) ThesaurusIterator

	// Contains checks if the given key exists in the thesaurus.
	Contains(key []byte) (bool, error)
}

Thesaurus encapsulates a structured collection of terms and their associated synonyms.

type ThesaurusIterator

type ThesaurusIterator interface {
	// Next returns the next entry in the thesaurus or an error if iteration fails.
	Next() (*index.ThesaurusEntry, error)
}

ThesaurusIterator iterates over terms in a thesaurus.

type ThesaurusSegment

type ThesaurusSegment interface {
	Segment
	// Thesaurus returns the Thesaurus with the specified name.
	Thesaurus(name string) (Thesaurus, error)
}

ThesaurusSegment provides access to a thesaurus within a specific segment of the index.

type UnpersistedSegment

type UnpersistedSegment interface {
	Segment
	Persist(path string) error
}