Documentation
¶
Overview ¶
Package store is a JSON document store on top of BadgerDB.
Keys are strings of the form "prefix:id"; values are the JSON encoding of the stored object. Account-scoped helpers prepend an account ID to build "{accountID}:{prefix}:{key}" for multi-tenant data.
Index ¶
- func ScopedKey(accountID, prefix, key string) string
- func ScopedPrefix(accountID, prefix string) string
- type BadgerLogger
- type Filter
- type LevelMetrics
- type Logger
- type Metrics
- type Options
- type Query
- type Store
- func (store *Store) Close() error
- func (s *Store) CollectMetrics() *Metrics
- func (store *Store) DeleteByMeta(meta byte) error
- func (store *Store) DeleteByPrefix(prefix string) error
- func (store *Store) DeleteKeys(keysForDelete [][]byte) error
- func (store *Store) DeleteOneByKey(key string) error
- func (store *Store) DeleteScoped(accountID, prefix, key string) error
- func (store *Store) DeleteScopedByPrefix(accountID, prefix string) error
- func (store *Store) GarbageCollection() error
- func (store *Store) Get(prefix string, v interface{}) error
- func (store *Store) GetByMeta(meta byte, v interface{}) error
- func (store *Store) GetCount(query *Query) (int, error)
- func (store *Store) GetKeyExists(key string) (bool, error)
- func (store *Store) GetMetrics() *Metrics
- func (store *Store) GetOne(key string, obj interface{}) error
- func (store *Store) GetOneByFilter(prefix string, filters *[]Filter, obj interface{}) error
- func (store *Store) GetOneByFilterRaw(prefix string, filter Filter) (string, error)
- func (store *Store) GetOneRaw(key string) (string, error)
- func (store *Store) GetScoped(accountID, prefix, key string, obj interface{}) error
- func (store *Store) GetScopedByFilter(accountID, prefix string, filters *[]Filter, obj interface{}) error
- func (store *Store) GetScopedCount(accountID string, query *Query) (int, error)
- func (store *Store) GetScopedStream(w io.Writer, wg *sync.WaitGroup, accountID string, query *Query)
- func (store *Store) GetScopedStreamProjected(w io.Writer, wg *sync.WaitGroup, accountID string, query *Query, ...)
- func (store *Store) GetStream(w io.Writer, wg *sync.WaitGroup, query *Query)
- func (store *Store) GetStreamProjected(w io.Writer, wg *sync.WaitGroup, query *Query, ...)
- func (store *Store) GetWithQuery(query *Query, v interface{}) error
- func (store *Store) ListScoped(accountID, prefix string, v interface{}) error
- func (store *Store) ListScopedWithQuery(accountID string, query *Query, v interface{}) error
- func (store *Store) PutJSONPipe(prefix string, stream *io.PipeReader) error
- func (store *Store) PutOne(key string, obj interface{}) error
- func (store *Store) PutOneRaw(key string, obj string) error
- func (store *Store) PutOneRawWithTTL(key string, obj string, ttl time.Duration) error
- func (store *Store) PutOneRawWithTTLAndMeta(key string, obj string, ttl time.Duration, meta byte) error
- func (store *Store) PutOneWithTTL(key string, obj interface{}, ttl time.Duration) error
- func (store *Store) PutOneWithTTLAndMeta(key string, obj interface{}, ttl time.Duration, meta byte) error
- func (store *Store) PutScoped(accountID, prefix, key string, obj interface{}) error
- func (store *Store) PutScopedWithTTL(accountID, prefix, key string, obj interface{}, ttl time.Duration) error
- func (store *Store) PutSlice(prefix string, id string, objs interface{}) error
- func (store *Store) PutStream(prefix string, r io.Reader) error
- func (store *Store) StartGC(ctx context.Context, interval time.Duration)
- type TimeRange
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ScopedPrefix ¶
ScopedPrefix builds an account-scoped prefix for scanning: {accountID}:{prefix}:
Types ¶
type BadgerLogger ¶
type BadgerLogger struct {
// contains filtered or unexported fields
}
BadgerLogger adapts a Logger to badger's Logger interface.
func NewBadgerLogger ¶
func NewBadgerLogger(l Logger) *BadgerLogger
NewBadgerLogger creates a new badger-compatible logger.
func (*BadgerLogger) Debugf ¶
func (bl *BadgerLogger) Debugf(format string, args ...interface{})
Debugf implements badger.Logger.
func (*BadgerLogger) Errorf ¶
func (bl *BadgerLogger) Errorf(format string, args ...interface{})
Errorf implements badger.Logger.
func (*BadgerLogger) Infof ¶
func (bl *BadgerLogger) Infof(format string, args ...interface{})
Infof implements badger.Logger.
func (*BadgerLogger) Warningf ¶
func (bl *BadgerLogger) Warningf(format string, args ...interface{})
Warningf implements badger.Logger.
type LevelMetrics ¶
type LevelMetrics struct {
Level int `json:"level"`
NumTables int `json:"num_tables"`
Size int64 `json:"size"`
}
LevelMetrics contains per-level table information from BadgerDB.
type Logger ¶
Logger is the minimal logging surface the store needs. Any logger with these four methods can be passed in Options; when none is given the store builds a zerolog console logger.
func NewZerologLogger ¶
NewZerologLogger wraps a zerolog.Logger as a store Logger.
type Metrics ¶
type Metrics struct {
SizeLsm int64 `json:"size_lsm"`
SizeVLog int64 `json:"size_vlog"`
SizeTotal int64 `json:"size_total"`
NumKeys int64 `json:"num_keys"`
NumLevels int `json:"num_levels"`
NumTables int `json:"num_tables"`
LevelSizes []LevelMetrics `json:"level_sizes"`
BlockCacheHits int64 `json:"block_cache_hits"`
BlockCacheMisses int64 `json:"block_cache_misses"`
DiskTotal uint64 `json:"disk_total"`
DiskFree uint64 `json:"disk_free"`
DiskUsedPct float64 `json:"disk_used_pct"`
PutOps int64 `json:"put_ops"`
GetOps int64 `json:"get_ops"`
DeleteOps int64 `json:"delete_ops"`
}
Metrics contains metadata about the store.
func NewMetrics ¶
NewMetrics creates a basic Metrics with LSM and VLog sizes (backward compat).
type Options ¶
type Options struct {
Location string `json:"location"`
Application string `json:"application"`
Component string `json:"component"`
IndexCacheSize int64 `json:"index_cache_size"`
BlockCacheSize int64 `json:"block_cache_size"`
Logger Logger `json:"-"`
}
Options configures a Store. Application and Component are only used to tag the default logger when Logger is nil.
type Query ¶
type Query struct {
Prefix string `json:"prefix"`
Filters []*Filter `json:"filters"`
Limit int `json:"limit"`
Offset int `json:"offset"`
TimeRange TimeRange `json:"time_range"`
}
Query describes a prefix scan with optional filters and paging.
type Store ¶
type Store struct {
DB *badger.DB
Logger Logger
Location string
// contains filtered or unexported fields
}
Store wraps a BadgerDB instance.
func (*Store) CollectMetrics ¶
CollectMetrics gathers comprehensive metrics from the store.
func (*Store) DeleteByMeta ¶
DeleteByMeta deletes every key whose user meta byte equals meta, then runs value-log garbage collection.
func (*Store) DeleteByPrefix ¶
DeleteByPrefix deletes every key under prefix. Keys are collected in a read transaction first, then deleted, to avoid iterating while mutating.
func (*Store) DeleteKeys ¶
DeleteKeys deletes the given keys, committing and starting a new transaction whenever badger reports ErrTxnTooBig.
func (*Store) DeleteOneByKey ¶
DeleteOneByKey deletes a single key.
func (*Store) DeleteScoped ¶
DeleteScoped deletes a value by account-scoped key.
func (*Store) DeleteScopedByPrefix ¶
DeleteScopedByPrefix deletes all values for an account-scoped prefix.
func (*Store) GarbageCollection ¶
GarbageCollection runs one round of value-log GC. Badger's "nothing to rewrite" result is not an error and is not reported as one.
func (*Store) Get ¶
Get unmarshals every value under prefix into v, which must be a pointer to a slice.
func (*Store) GetCount ¶
GetCount returns the number of values under "prefix:" matching the filters.
func (*Store) GetKeyExists ¶
GetKeyExists reports whether the raw value stored at key is "true". A missing key reports false with no error; any other error (I/O failure, closed DB) is returned so callers can distinguish "absent" from "could not read".
func (*Store) GetMetrics ¶
GetMetrics returns basic size metrics. Prefer CollectMetrics.
func (*Store) GetOneByFilter ¶
GetOneByFilter unmarshals the first value under "prefix:" matching all filters into obj. obj is left untouched when nothing matches.
func (*Store) GetOneByFilterRaw ¶
GetOneByFilterRaw returns the first raw value under "prefix:" matching filter.
func (*Store) GetScopedByFilter ¶
func (store *Store) GetScopedByFilter(accountID, prefix string, filters *[]Filter, obj interface{}) error
GetScopedByFilter retrieves a single value by account-scoped prefix and filters.
func (*Store) GetScopedCount ¶
GetScopedCount returns the count of items for an account-scoped prefix.
func (*Store) GetScopedStream ¶
func (store *Store) GetScopedStream(w io.Writer, wg *sync.WaitGroup, accountID string, query *Query)
GetScopedStream streams values for an account-scoped prefix.
func (*Store) GetScopedStreamProjected ¶
func (store *Store) GetScopedStreamProjected(w io.Writer, wg *sync.WaitGroup, accountID string, query *Query, transform func(value []byte) ([]byte, bool))
GetScopedStreamProjected streams an account-scoped prefix with a projection. See GetStreamProjected.
func (*Store) GetStream ¶
GetStream writes a JSON array of matching values to w on a goroutine that is tracked by wg. Query.Prefix is scanned as "prefix:".
func (*Store) GetStreamProjected ¶
func (store *Store) GetStreamProjected(w io.Writer, wg *sync.WaitGroup, query *Query, transform func(value []byte) ([]byte, bool))
GetStreamProjected streams a JSON array of transformed values under "query.Prefix:" to w on a goroutine tracked by wg, mirroring GetStream but applying transform to each stored value before writing. transform returns the bytes to emit and whether to include the item (false skips it) -- typically a projection to a few fields (for example map markers: id/lat/lon) so the wire payload is a fraction of the full documents. Emitted bytes must be valid JSON. Filters, Offset, and Limit are honored as in GetStream.
This uses a single iterator deliberately: benchmarks showed Badger's parallel Stream framework is slower and allocates ~19x more memory for prefix scans of our size (it targets full-DB backup across many SSTables), so the lean single-iterator scan wins here.
func (*Store) GetWithQuery ¶
GetWithQuery unmarshals every value under query.Prefix that matches all of query.Filters (and the time range, if set) into v, honoring query.Offset and query.Limit for pagination.
func (*Store) ListScoped ¶
ListScoped returns all values for an account-scoped prefix.
func (*Store) ListScopedWithQuery ¶
ListScopedWithQuery returns values for an account-scoped prefix with query filters.
func (*Store) PutJSONPipe ¶
func (store *Store) PutJSONPipe(prefix string, stream *io.PipeReader) error
PutJSONPipe reads a JSON array of objects from stream and stores each under "prefix:<uuid>". Objects without a uuid field are skipped and logged.
func (*Store) PutOneRawWithTTL ¶
PutOneRawWithTTL stores a string value verbatim with an expiry.
func (*Store) PutOneRawWithTTLAndMeta ¶
func (store *Store) PutOneRawWithTTLAndMeta(key string, obj string, ttl time.Duration, meta byte) error
PutOneRawWithTTLAndMeta stores a string value verbatim with an expiry and meta byte.
func (*Store) PutOneWithTTL ¶
PutOneWithTTL stores obj as JSON under key with an expiry.
func (*Store) PutOneWithTTLAndMeta ¶
func (store *Store) PutOneWithTTLAndMeta(key string, obj interface{}, ttl time.Duration, meta byte) error
PutOneWithTTLAndMeta stores obj with an expiry and a user meta byte.
func (*Store) PutScopedWithTTL ¶
func (store *Store) PutScopedWithTTL(accountID, prefix, key string, obj interface{}, ttl time.Duration) error
PutScopedWithTTL stores a value with an account-scoped key and TTL.
func (*Store) PutSlice ¶
PutSlice stores every element of objs under "prefix:<id>", where id is the gjson path of the element's identifier field.
func (*Store) PutStream ¶
PutStream reads a JSON array of objects from r and stores each under "prefix:<uuid>". The reader is typically an http.Response.Body. A malformed stream returns an error; individual objects that fail to store are logged and skipped.
func (*Store) StartGC ¶
StartGC runs value-log garbage collection on the given interval until ctx is cancelled, returning immediately (GC runs on its own goroutine). Badger only reclaims value-log space when RunValueLogGC is called, so a long-running service should start this once after opening the store. Each tick runs GC repeatedly until badger reports nothing left to rewrite.