docstore

Documentation

Overview

Package docstore implements a JSON-based document store built on top of the Versioned Key-Value store and the Blob store.

Each document will get assigned a MongoDB like ObjectId:

<binary encoded uint32 (4 bytes) + 8 random bytes hex encoded >

The resulting id will have a length of 24 characters encoded as hex (12 raw bytes).

The JSON document will be stored directly inside the vkv entry.

docstore:<collection>:<id> => <flag (1 byte) + JSON blob>

Document will be automatically sorted by creation time thanks to the ID.

The raw JSON will be stored as is, but the API will add the _id and other special fields on the fly.

Index

• Variables
• func SetLuaGlobals(L *lua.LState)
• type DocStore
  • func New(logger log.Logger, conf *config.Config, kvStore store.KvStore, ...) (*DocStore, error)
  • func (docstore *DocStore) Close() error
  • func (docstore *DocStore) Collections() ([]string, error)
  • func (docstore *DocStore) Fetch(collection, sid string, res *map[string]interface{}, withSpecialFields bool, ...) (*id.ID, map[string]interface{}, error)
  • func (docstore *DocStore) FetchVersions(collection, sid string, start int64, limit int, fetchPointers bool) ([]map[string]interface{}, map[string]interface{}, int64, error)
  • func (dc *DocStore) GetSortIndex(col, name string) (Indexer, error)
  • func (dc *DocStore) GetSortIndexes(col string) ([]Indexer, error)
  • func (docstore *DocStore) IndexDoc(collection string, _id *id.ID, doc map[string]interface{}) error
  • func (docstore *DocStore) Insert(collection string, doc map[string]interface{}) (*id.ID, error)
  • func (docstore *DocStore) IterCollection(collection string, cb func(*id.ID, map[string]interface{}) error) error
  • func (docstore *DocStore) LuaQuery(L *lua.LState, lfunc *lua.LFunction, collection string, cursor string, ...) ([]map[string]interface{}, map[string]interface{}, string, *executionStats, ...)
  • func (dc *DocStore) LuaSetupSortIndex(col, name, field string) error
  • func (docstore *DocStore) LuaTextSearch(L *lua.LState) int
  • func (docstore *DocStore) Query(collection string, query *query, cursor string, limit int, asOf int64) ([]map[string]interface{}, map[string]interface{}, *executionStats, error)
  • func (docstore *DocStore) RebuildIndexes(collection string) error
  • func (docstore *DocStore) Register(r *mux.Router, basicAuth func(http.Handler) http.Handler)
  • func (docstore *DocStore) Remove(collection, sid string) (*id.ID, error)
  • func (docstore *DocStore) Update(collection, sid string, newDoc map[string]interface{}, ifMatch string) (*id.ID, error)
• type IDIterator
• type Indexer
• type LuaHook
  • func NewLuaHook(L *lua.LState, code string) (*LuaHook, error)
  • func (h *LuaHook) Execute(doc map[string]interface{}) (map[string]interface{}, error)
  • func (h *LuaHook) ExecuteNoResult(doc map[string]interface{}) error
  • func (h *LuaHook) ExecuteReduce(key string, docs []map[string]interface{}) (map[string]interface{}, error)
  • func (h *LuaHook) LFunction() *lua.LFunction
• type LuaQueryEngine
  • func (lqe *LuaQueryEngine) Close() error
  • func (lqe *LuaQueryEngine) Match(doc map[string]interface{}) (bool, error)
• type MapReduceEngine
  • func NewMapReduceEngine() *MapReduceEngine
  • func (mre *MapReduceEngine) Close()
  • func (mre *MapReduceEngine) Duplicate() (*MapReduceEngine, error)
  • func (mre *MapReduceEngine) Finalize() (map[string]map[string]interface{}, error)
  • func (mre *MapReduceEngine) Map(doc map[string]interface{}) error
  • func (mre *MapReduceEngine) Reduce(other *MapReduceEngine) error
  • func (mre *MapReduceEngine) SetupMap(code string) error
  • func (mre *MapReduceEngine) SetupReduce(code string) error
• type MatchAllEngine
  • func (mae *MatchAllEngine) Close() error
  • func (mae *MatchAllEngine) Match(_ map[string]interface{}) (bool, error)
• type QueryMatcher

Variables

var (
	PrefixIndexKeyFmt = "docstore-index:%s"
	IndexKeyFmt       = PrefixIndexKeyFmt + ":%s"
)

var ErrDocNotFound = errors.New("document not found")

var ErrPreconditionFailed = errors.New("precondition failed")

var ErrSortIndexInvalidNameOrField = errors.New("sort index invalid (bad name or field)")

var ErrSortIndexNotFound = errors.New("sort index not found")

var ErrUnprocessableEntity = errors.New("unprocessable entity")

ErrUnprocessableEntity is returned when a document is faulty

Functions

func SetLuaGlobals

func SetLuaGlobals(L *lua.LState)

Types

type DocStore

type DocStore struct {
	// contains filtered or unexported fields
}

DocStore holds the docstore manager

func New

func New(logger log.Logger, conf *config.Config, kvStore store.KvStore, blobStore store.BlobStore, ft *filetree.FileTree) (*DocStore, error)

New initializes the `DocStoreExt`

func (*DocStore) Close

func (docstore *DocStore) Close() error

Close closes all the open DB files.

func (*DocStore) Collections

func (docstore *DocStore) Collections() ([]string, error)

Collections returns all the existing collections

func (*DocStore) Fetch

func (docstore *DocStore) Fetch(collection, sid string, res *map[string]interface{}, withSpecialFields bool, fetchPointers bool, version int64) (*id.ID, map[string]interface{}, error)

Fetch a single document into `res` and returns the `id.ID`

func (*DocStore) FetchVersions

func (docstore *DocStore) FetchVersions(collection, sid string, start int64, limit int, fetchPointers bool) ([]map[string]interface{}, map[string]interface{}, int64, error)

FetchVersions returns all verions/revisions for the given doc ID

func (*DocStore) GetSortIndex

func (dc *DocStore) GetSortIndex(col, name string) (Indexer, error)

GetSortIndex lazy-loads a sort index

func (*DocStore) GetSortIndexes

func (dc *DocStore) GetSortIndexes(col string) ([]Indexer, error)

func (*DocStore) IndexDoc

func (docstore *DocStore) IndexDoc(collection string, _id *id.ID, doc map[string]interface{}) error

func (*DocStore) Insert

func (docstore *DocStore) Insert(collection string, doc map[string]interface{}) (*id.ID, error)

Insert the given doc (`*map[string]interface{}` for now) in the given collection

func (*DocStore) IterCollection

func (docstore *DocStore) IterCollection(collection string, cb func(*id.ID, map[string]interface{}) error) error

func (*DocStore) LuaQuery

func (docstore *DocStore) LuaQuery(L *lua.LState, lfunc *lua.LFunction, collection string, cursor string, sortIndex string, limit int) ([]map[string]interface{}, map[string]interface{}, string, *executionStats, error)

LuaQuery performs a Lua query

func (*DocStore) LuaSetupSortIndex

func (dc *DocStore) LuaSetupSortIndex(col, name, field string) error

func (*DocStore) LuaTextSearch

func (docstore *DocStore) LuaTextSearch(L *lua.LState) int

func (*DocStore) Query

func (docstore *DocStore) Query(collection string, query *query, cursor string, limit int, asOf int64) ([]map[string]interface{}, map[string]interface{}, *executionStats, error)

Query performs a query

func (*DocStore) RebuildIndexes

func (docstore *DocStore) RebuildIndexes(collection string) error

func (*DocStore) Register

func (docstore *DocStore) Register(r *mux.Router, basicAuth func(http.Handler) http.Handler)

Register registers all the HTTP handlers for the extension

func (*DocStore) Remove

func (docstore *DocStore) Remove(collection, sid string) (*id.ID, error)

func (*DocStore) Update

func (docstore *DocStore) Update(collection, sid string, newDoc map[string]interface{}, ifMatch string) (*id.ID, error)

type IDIterator

type IDIterator interface {
	Iter(collection string, cursor string, desc bool, fetchLimit int, asOf int64) (ids []*id.ID, nextCursor string, err error)
	Name() string
}

IDIterator is the interface that wraps the Iter method

Iter allow to iterates over all the valid document IDs for a given asOf (as <= 0 means "as of now")

type Indexer

type Indexer interface {
	Index(id *id.ID, doc map[string]interface{}) error
	io.Closer
	IDIterator
}

Indexer is the interface that wraps the Index method

type LuaHook

type LuaHook struct {
	L *lua.LState

	ID string
	// contains filtered or unexported fields
}

func NewLuaHook

func NewLuaHook(L *lua.LState, code string) (*LuaHook, error)

func (*LuaHook) Execute

func (h *LuaHook) Execute(doc map[string]interface{}) (map[string]interface{}, error)

TODO(tsileo): helper for validation like for required fields and returns details for 422 error (field error details)

func (*LuaHook) ExecuteNoResult

func (h *LuaHook) ExecuteNoResult(doc map[string]interface{}) error

func (*LuaHook) ExecuteReduce

func (h *LuaHook) ExecuteReduce(key string, docs []map[string]interface{}) (map[string]interface{}, error)

func (*LuaHook) LFunction

func (h *LuaHook) LFunction() *lua.LFunction

type LuaQueryEngine

type LuaQueryEngine struct {
	L *lua.LState // Lua state that will live the whole query
	// contains filtered or unexported fields
}

func (*LuaQueryEngine) Close

func (lqe *LuaQueryEngine) Close() error

func (*LuaQueryEngine) Match

func (lqe *LuaQueryEngine) Match(doc map[string]interface{}) (bool, error)

type MapReduceEngine

type MapReduceEngine struct {
	L *lua.LState

	M *LuaHook // Map
	R *LuaHook // Reduce

	sync.Mutex
	// contains filtered or unexported fields
}

func NewMapReduceEngine

func NewMapReduceEngine() *MapReduceEngine

func (*MapReduceEngine) Close

func (mre *MapReduceEngine) Close()

func (*MapReduceEngine) Duplicate

func (mre *MapReduceEngine) Duplicate() (*MapReduceEngine, error)

Duplicate returns a new `MapReduceEngine` with the same map and reduce hook as the current instance.

func (*MapReduceEngine) Finalize

func (mre *MapReduceEngine) Finalize() (map[string]map[string]interface{}, error)

func (*MapReduceEngine) Map

func (mre *MapReduceEngine) Map(doc map[string]interface{}) error

func (*MapReduceEngine) Reduce

func (mre *MapReduceEngine) Reduce(other *MapReduceEngine) error

other can be an already closed engine

func (*MapReduceEngine) SetupMap

func (mre *MapReduceEngine) SetupMap(code string) error

SetupMap loads the map function (as a string, the code must return a function)

func (*MapReduceEngine) SetupReduce

func (mre *MapReduceEngine) SetupReduce(code string) error

SetupReduce loads the reduce function (as a string, the code must return a function)

type MatchAllEngine

type MatchAllEngine struct{}

func (*MatchAllEngine) Close

func (mae *MatchAllEngine) Close() error

func (*MatchAllEngine) Match

func (mae *MatchAllEngine) Match(_ map[string]interface{}) (bool, error)

type QueryMatcher

type QueryMatcher interface {
	Match(map[string]interface{}) (bool, error)
	Close() error
}