segment

package
v1.0.14 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 8, 2020 License: Apache-2.0 Imports: 9 Imported by: 104

Documentation

Index

Constants

View Source 
const (
	MaxVarintSize = 9

	// IntMin is chosen such that the range of int tags does not overlap the
	// ascii character set that is frequently used in testing.
	IntMin = 0x80 // 128

	// IntMax is the maximum int tag value.
	IntMax = 0xfd // 253
)

Variables

View Source 
var AnEmptyPostingsIterator = &EmptyPostingsIterator{}
View Source 
var ErrClosed = fmt.Errorf("index closed")
View Source 
var ErrMemUvarintReaderOverflow = errors.New("MemUvarintReader overflow")

Functions

func DecodeUvarintAscending 

func DecodeUvarintAscending(b []byte) ([]byte, uint64, error)

DecodeUvarintAscending decodes a varint encoded uint64 from the input buffer. The remainder of the input buffer and the decoded uint64 are returned.

func EncodeUvarintAscending 

func EncodeUvarintAscending(b []byte, v uint64) []byte

EncodeUvarintAscending encodes the uint64 value using a variable length (length-prefixed) representation. The length is encoded as a single byte indicating the number of encoded bytes (-8) to follow. See EncodeVarintAscending for rationale. The encoded bytes are appended to the supplied buffer and the final buffer is returned.

func IncrementBytes added in v0.8.0 

func IncrementBytes(in []byte) []byte

func LiteralPrefix added in v0.8.0 

func LiteralPrefix(s *syntax.Regexp) string

Returns the literal prefix given the parse tree for a regexp

func ParseRegexp added in v0.8.0 

func ParseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, err error)

Types

type DictionaryIterator 

type DictionaryIterator interface {
	Next() (*index.DictEntry, error)
}

type DocVisitState added in v0.8.0 

type DocVisitState interface {
}

type DocumentFieldTermVisitable 

type DocumentFieldTermVisitable interface {
	VisitDocumentFieldTerms(localDocNum uint64, fields []string,
		visitor index.DocumentFieldTermVisitor, optional DocVisitState) (DocVisitState, error)

	// VisitableDocValueFields implementation should return
	// the list of fields which are document value persisted and
	// therefore visitable by the above VisitDocumentFieldTerms method.
	VisitableDocValueFields() ([]string, error)
}

DocumentFieldTermVisitable is implemented by various scorch segment implementations with persistence for the un inverting of the postings or other indexed values.

type DocumentFieldValueVisitor 

type DocumentFieldValueVisitor func(field string, typ byte, value []byte, pos []uint64) bool

DocumentFieldValueVisitor defines a callback to be visited for each stored field value. The return value determines if the visitor should keep going. Returning true continues visiting, false stops.

type EmptyDictionary 

type EmptyDictionary struct{}

func (*EmptyDictionary) AutomatonIterator added in v0.8.0 

func (e *EmptyDictionary) AutomatonIterator(a vellum.Automaton,
	startKeyInclusive, endKeyExclusive []byte) DictionaryIterator

func (*EmptyDictionary) Contains added in v0.8.0 

func (e *EmptyDictionary) Contains(key []byte) (bool, error)

func (*EmptyDictionary) Iterator 

func (e *EmptyDictionary) Iterator() DictionaryIterator

func (*EmptyDictionary) OnlyIterator added in v0.8.0 

func (e *EmptyDictionary) OnlyIterator(onlyTerms [][]byte,
	includeCount bool) DictionaryIterator

func (*EmptyDictionary) PostingsList 

func (e *EmptyDictionary) PostingsList(term []byte,
	except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error)

func (*EmptyDictionary) PrefixIterator 

func (e *EmptyDictionary) PrefixIterator(prefix string) DictionaryIterator

func (*EmptyDictionary) RangeIterator 

func (e *EmptyDictionary) RangeIterator(start, end string) DictionaryIterator

type EmptyDictionaryIterator 

type EmptyDictionaryIterator struct{}

func (*EmptyDictionaryIterator) Contains added in v0.8.0 

func (e *EmptyDictionaryIterator) Contains(key []byte) (bool, error)

func (*EmptyDictionaryIterator) Next 

func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error)

type EmptyPostingsIterator 

type EmptyPostingsIterator struct{}

func (*EmptyPostingsIterator) Advance added in v0.8.0 

func (e *EmptyPostingsIterator) Advance(uint64) (Posting, error)

func (*EmptyPostingsIterator) Next 

func (e *EmptyPostingsIterator) Next() (Posting, error)

func (*EmptyPostingsIterator) Size added in v0.8.0 

func (e *EmptyPostingsIterator) Size() int

type EmptyPostingsList 

type EmptyPostingsList struct{}

func (*EmptyPostingsList) Count 

func (e *EmptyPostingsList) Count() uint64

func (*EmptyPostingsList) Iterator 

func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool,
	prealloc PostingsIterator) PostingsIterator

func (*EmptyPostingsList) Size added in v0.8.0 

func (e *EmptyPostingsList) Size() int

type EmptySegment 

type EmptySegment struct{}

func (*EmptySegment) AddRef 

func (e *EmptySegment) AddRef()

func (*EmptySegment) Close 

func (e *EmptySegment) Close() error

func (*EmptySegment) Count 

func (e *EmptySegment) Count() uint64

func (*EmptySegment) DecRef 

func (e *EmptySegment) DecRef() error

func (*EmptySegment) Dictionary 

func (e *EmptySegment) Dictionary(field string) (TermDictionary, error)

func (*EmptySegment) DocID added in v0.8.0 

func (e *EmptySegment) DocID(num uint64) ([]byte, error)

func (*EmptySegment) DocNumbers 

func (e *EmptySegment) DocNumbers([]string) (*roaring.Bitmap, error)

func (*EmptySegment) Fields 

func (e *EmptySegment) Fields() []string

func (*EmptySegment) Size added in v0.8.0 

func (e *EmptySegment) Size() uint64

func (*EmptySegment) VisitDocument 

func (e *EmptySegment) VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error

type Location 

type Location interface {
	Field() string
	Start() uint64
	End() uint64
	Pos() uint64
	ArrayPositions() []uint64
	Size() int
}

type MemUvarintReader added in v0.8.0 

type MemUvarintReader struct {
	C int // index of next byte to read from S
	S []byte
}

func NewMemUvarintReader added in v0.8.0 

func NewMemUvarintReader(s []byte) *MemUvarintReader

func (*MemUvarintReader) Len added in v0.8.0 

func (r *MemUvarintReader) Len() int

Len returns the number of unread bytes.

func (*MemUvarintReader) ReadUvarint added in v0.8.0 

func (r *MemUvarintReader) ReadUvarint() (uint64, error)

ReadUvarint reads an encoded uint64. The original code this was based on is at encoding/binary/ReadUvarint().

func (*MemUvarintReader) Reset added in v0.8.0 

func (r *MemUvarintReader) Reset(s []byte)

func (*MemUvarintReader) SkipBytes added in v0.8.0 

func (r *MemUvarintReader) SkipBytes(count int)

SkipBytes skips a count number of bytes.

func (*MemUvarintReader) SkipUvarint added in v0.8.0 

func (r *MemUvarintReader) SkipUvarint()

SkipUvarint skips ahead one encoded uint64.

type OptimizablePostingsIterator added in v1.0.2 

type OptimizablePostingsIterator interface {
	ActualBitmap() *roaring.Bitmap
	DocNum1Hit() (uint64, bool)
	ReplaceActual(*roaring.Bitmap)
}

type PersistedSegment added in v1.0.2 

type PersistedSegment interface {
	Segment
	Path() string
}

type Plugin added in v1.0.2 

type Plugin interface {

	// Type is the name for this segment plugin
	Type() string

	// Version is a numeric value identifying a specific version of this type.
	// When incompatible changes are made to a particular type of plugin, the
	// version must be incremented.
	Version() uint32

	// New takes a set of AnalysisResults and turns them into a new Segment
	New(results []*index.AnalysisResult) (Segment, uint64, error)

	// Open attempts to open the file at the specified path and
	// return the corresponding Segment
	Open(path string) (Segment, error)

	// Merge takes a set of Segments, and creates a new segment on disk at
	// the specified path.
	// Drops is a set of bitmaps (one for each segment) indicating which
	// documents can be dropped from the segments during the merge.
	// If the closeCh channel is closed, Merge will cease doing work at
	// the next opportunity, and return an error (closed).
	// StatsReporter can optionally be provided, in which case progress
	// made during the merge is reported while operation continues.
	// Returns:
	// A slice of new document numbers (one for each input segment),
	// this allows the caller to know a particular document's new
	// document number in the newly merged segment.
	// The number of bytes written to the new segment file.
	// An error, if any occurred.
	Merge(segments []Segment, drops []*roaring.Bitmap, path string,
		closeCh chan struct{}, s StatsReporter) (
		[][]uint64, uint64, error)
}

Plugin represents the essential functions required by a package to plug in it's segment implementation

type Posting 

type Posting interface {
	Number() uint64

	Frequency() uint64
	Norm() float64

	Locations() []Location

	Size() int
}

type PostingsIterator 

type PostingsIterator interface {
	// The caller is responsible for copying whatever it needs from
	// the returned Posting instance before calling Next(), as some
	// implementations may return a shared instance to reduce memory
	// allocations.
	Next() (Posting, error)

	// Advance will return the posting with the specified doc number
	// or if there is no such posting, the next posting.
	// Callers MUST NOT attempt to pass a docNum that is less than or
	// equal to the currently visited posting doc Num.
	Advance(docNum uint64) (Posting, error)

	Size() int
}

func NewUnadornedPostingsIteratorFrom1Hit added in v1.0.2 

func NewUnadornedPostingsIteratorFrom1Hit(docNum1Hit uint64) PostingsIterator

func NewUnadornedPostingsIteratorFromBitmap added in v1.0.2 

func NewUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) PostingsIterator

type PostingsList 

type PostingsList interface {
	Iterator(includeFreq, includeNorm, includeLocations bool, prealloc PostingsIterator) PostingsIterator

	Size() int

	Count() uint64
}

type Segment 

type Segment interface {
	Dictionary(field string) (TermDictionary, error)

	VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error

	DocID(num uint64) ([]byte, error)

	Count() uint64

	DocNumbers([]string) (*roaring.Bitmap, error)

	Fields() []string

	Close() error

	Size() int

	AddRef()
	DecRef() error
}

type StatsReporter added in v0.8.0 

type StatsReporter interface {
	ReportBytesWritten(bytesWritten uint64)
}

type TermDictionary 

type TermDictionary interface {
	PostingsList(term []byte, except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error)

	Iterator() DictionaryIterator
	PrefixIterator(prefix string) DictionaryIterator
	RangeIterator(start, end string) DictionaryIterator
	AutomatonIterator(a vellum.Automaton,
		startKeyInclusive, endKeyExclusive []byte) DictionaryIterator
	OnlyIterator(onlyTerms [][]byte, includeCount bool) DictionaryIterator

	Contains(key []byte) (bool, error)
}

type UnadornedPosting added in v1.0.2 

type UnadornedPosting uint64

func (UnadornedPosting) Frequency added in v1.0.2 

func (p UnadornedPosting) Frequency() uint64

func (UnadornedPosting) Locations added in v1.0.2 

func (p UnadornedPosting) Locations() []Location

func (UnadornedPosting) Norm added in v1.0.2 

func (p UnadornedPosting) Norm() float64

func (UnadornedPosting) Number added in v1.0.2 

func (p UnadornedPosting) Number() uint64

func (UnadornedPosting) Size added in v1.0.2 

func (p UnadornedPosting) Size() int

type UnadornedPostingsIterator1Hit added in v1.0.2 

type UnadornedPostingsIterator1Hit struct {
	// contains filtered or unexported fields
}

func (*UnadornedPostingsIterator1Hit) Advance added in v1.0.2 

func (i *UnadornedPostingsIterator1Hit) Advance(docNum uint64) (Posting, error)

func (*UnadornedPostingsIterator1Hit) Next added in v1.0.2 

func (i *UnadornedPostingsIterator1Hit) Next() (Posting, error)

func (*UnadornedPostingsIterator1Hit) Size added in v1.0.2 

func (i *UnadornedPostingsIterator1Hit) Size() int

type UnadornedPostingsIteratorBitmap added in v1.0.2 

type UnadornedPostingsIteratorBitmap struct {
	// contains filtered or unexported fields
}

func (*UnadornedPostingsIteratorBitmap) ActualBitmap added in v1.0.10 

func (i *UnadornedPostingsIteratorBitmap) ActualBitmap() *roaring.Bitmap

func (*UnadornedPostingsIteratorBitmap) Advance added in v1.0.2 

func (i *UnadornedPostingsIteratorBitmap) Advance(docNum uint64) (Posting, error)

func (*UnadornedPostingsIteratorBitmap) DocNum1Hit added in v1.0.10 

func (i *UnadornedPostingsIteratorBitmap) DocNum1Hit() (uint64, bool)

func (*UnadornedPostingsIteratorBitmap) Next added in v1.0.2 

func (i *UnadornedPostingsIteratorBitmap) Next() (Posting, error)

func (*UnadornedPostingsIteratorBitmap) ReplaceActual added in v1.0.10 

func (i *UnadornedPostingsIteratorBitmap) ReplaceActual(actual *roaring.Bitmap)

func (*UnadornedPostingsIteratorBitmap) Size added in v1.0.2 

func (i *UnadornedPostingsIteratorBitmap) Size() int

type UnpersistedSegment added in v1.0.2 

type UnpersistedSegment interface {
	Segment
	Persist(path string) error
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.