data

Documentation

Index

• Constants
• Variables
• func DisOrFirst(tie float32, q []iq.Query) iq.Query
• func OrOrFirst(q []iq.Query) iq.Query
• type Document
  • func (d *Document) DocumentID() int32
  • func (d *Document) IndexableFields() map[string][]string
• type Shard
  • func NewShard(root string, kind string, id uint32) *Shard
  • func (s *Shard) BulkUpsert(batch []*Document)
  • func (s *Shard) Close()
  • func (s *Shard) MakeQuery(field string, query string) iq.Query
  • func (s *Shard) ReadWeight(did int32) int32
  • func (s *Shard) Reindex(batchSize int)
  • func (s *Shard) Upsert(tx *gorm.DB, in *Document) *Document
  • func (s *Shard) WriteWeight(did int32, w int32) error
• type Store
  • func NewStore(root string, kind string) *Store
  • func (s *Store) BulkUpsert(batch []*Document)
  • func (s *Store) Close()
  • func (s *Store) Parallel(cb func(int, *Shard))
  • func (s *Store) Reindex(batchSize int)
  • func (s *Store) ShardFor(objectID string) *Shard
  • func (s *Store) ShardID(objectID string) uint32

Constants

const MAX_CHUNKS = 32

Variables

var DefaultAnalyzer = analyzer.NewAnalyzer(DefaultNormalizer, DefaultSearchTokenizer, DefaultIndexTokenizer)

var DefaultIndexTokenizer = []tokenize.Tokenizer{
	tokenize.NewCustom(prefixLine),
	tokenize.NewUnique(),
}

haha this is extreme hack

var DefaultNormalizer = []norm.Normalizer{norm.NewCustom(ascii)}

var DefaultSearchTokenizer = []tokenize.Tokenizer{
	tokenize.NewWhitespace(),
}

var MAX_TOKEN_SIZE = 10

var N_SHARDS = uint32(16)

having 16 shards for stackoverflow means 2.5 million documents per shard

Functions

func DisOrFirst

func DisOrFirst(tie float32, q []iq.Query) iq.Query

func OrOrFirst

func OrOrFirst(q []iq.Query) iq.Query

Types

type Document

type Document struct {
	RowID    int32  `gorm:"primary_key;auto_increment:true;not null"`
	ObjectID string `gorm:"unique_index"`

	Title string `json:"title,omitempty"`
	Body  []byte `json:"body,omitempty"`
	Tags  string `json:"tags,omitempty"`

	Popularity int   `json:"popularity,omitempty"`
	Indexed    int32 `gorm:"index:idx_indexed"`
}

func (*Document) DocumentID

func (d *Document) DocumentID() int32

func (*Document) IndexableFields

func (d *Document) IndexableFields() map[string][]string

type Shard

type Shard struct {
	DB     *gorm.DB
	Dir    *inv.LiteIndex
	Weight *os.File
	ID     uint32
}

func NewShard

func NewShard(root string, kind string, id uint32) *Shard

func (*Shard) BulkUpsert

func (s *Shard) BulkUpsert(batch []*Document)

func (*Shard) Close

func (s *Shard) Close()

func (*Shard) MakeQuery

func (s *Shard) MakeQuery(field string, query string) iq.Query

func (*Shard) ReadWeight

func (s *Shard) ReadWeight(did int32) int32

func (*Shard) Reindex

func (s *Shard) Reindex(batchSize int)

func (*Shard) Upsert

func (s *Shard) Upsert(tx *gorm.DB, in *Document) *Document

func (*Shard) WriteWeight

func (s *Shard) WriteWeight(did int32, w int32) error

type Store

type Store struct {
	Shards []*Shard
}

func NewStore

func NewStore(root string, kind string) *Store

func (*Store) BulkUpsert

func (s *Store) BulkUpsert(batch []*Document)

func (*Store) Close

func (s *Store) Close()

func (*Store) Parallel

func (s *Store) Parallel(cb func(int, *Shard))

func (*Store) Reindex

func (s *Store) Reindex(batchSize int)

func (*Store) ShardFor

func (s *Store) ShardFor(objectID string) *Shard

func (*Store) ShardID

func (s *Store) ShardID(objectID string) uint32