sdb

Documentation

Overview

Package sdb provides a simple and embeddable database with full text search support.

Example

package main

import (
	"errors"
	"fmt"
	"io/ioutil"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/analysis/analyzer/keyword"
	"nt.web.ve/go/sdb/pkg/sdb"
)

type Vehicle struct {
	ID    int
	Type  string
	Brand string
	Model string
}

type Person struct {
	ID, Name string
	Email    string
	Alive    bool
	Numbers  []int
	Vehicle  Vehicle
	Family   []Person

	Data map[string]interface{}

	Doctype string // Search index document type.
}

func main() {
	dir, err := ioutil.TempDir("", "sdb")
	if err != nil {
		panic(err)
	}

	opts := sdb.DefaultOptions(dir)

	// Advanced document mapping

	keywordField := bleve.NewTextFieldMapping()
	keywordField.Analyzer = keyword.Name

	peopleMapping := bleve.NewDocumentMapping()
	peopleMapping.AddFieldMappingsAt("Doctype", keywordField)
	// Without this, 'rrg' would match with 'ntrrg', 'atrrg', etc...
	peopleMapping.AddFieldMappingsAt("ID", keywordField)
	peopleMapping.AddFieldMappingsAt("Email", keywordField)
	// Without this, boolean fields couldn't be compared with 'true' of 'false'
	peopleMapping.AddFieldMappingsAt("Alive", keywordField)

	opts.Bleve.DocMappings["people"] = peopleMapping

	db, err := sdb.OpenWith(opts)
	if err != nil {
		panic(err)
	}

	// If no advanced options are needed, all the previous lines could be
	// replaced by:
	//
	//   db, err := sdb.Open("/path/to/database")
	//   if err != nil {
	//     panic(err)
	//   }

	defer db.Close()

	writeData(db)
	getData(db)
	deleteData(db)

}

func writeData(db *sdb.DB) {
	tx := db.NewTx(sdb.RW)
	defer tx.Discard()

	for _, p := range people {
		p := p
		if err := tx.Set([]byte(p.ID), &p); err != nil {
			panic(err)
		}
	}

	if err := tx.Commit(); err != nil {
		panic(err)
	}
}

func getData(db *sdb.DB) {
	tx := db.NewTx(sdb.RW)
	defer tx.Discard()

	p := Person{}
	if err := tx.Get([]byte("ntrrg"), &p); err != nil {
		panic(err)
	}

	fmt.Printf("Get -> %s: %s\n", p.ID, p.Name)

	q := "Email:ntrrg@example.com" // Any document with the given email

	keys, err := tx.Find(q, nil)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Find -> (%s): %q\n", q, keys)

	q = `Data.anime:"One Piece"` // Any document with One Piece in its anime list

	keys, err = tx.Find(q, &sdb.FindOptions{Sort: "ID", Limit: 2})
	if err != nil {
		panic(err)
	}

	fmt.Printf("Find -> (%s Sorted:ID Limit:2): %q\n", q, keys)

	prefix := []byte("nt") // Any key starting with "nt"
	keys = tx.Prefix(prefix)
	fmt.Printf("Prefix -> (%s): %q\n", prefix, keys)
}

func deleteData(db *sdb.DB) {
	tx := db.NewTx(sdb.RW)
	defer tx.Discard()

	if err := tx.Delete([]byte("ntrrg")); err != nil {
		panic(err)
	}

	p := Person{}
	if err := tx.Get([]byte("ntrrg"), &p); errors.Is(err, sdb.ErrKeyNotFound) {
		p.ID = "ntrrg"
		p.Name = "Not found"
	} else if err != nil {
		panic(err)
	}

	if err := tx.Commit(); err != nil {
		panic(err)
	}

	fmt.Printf("Delete -> %s: %s\n", p.ID, p.Name)
}

var people = []Person{
	{
		ID:    "ntrrg",
		Name:  "Rivera Notararigo Miguel Angel",
		Email: "ntrrg@example.com",
		Alive: true,

		Numbers: []int{0, 2, 11},

		Vehicle: Vehicle{
			ID:    1,
			Type:  "Car",
			Brand: "Toyota",
			Model: "Corolla Araya",
		},

		Family: []Person{
			{ID: "alirio", Name: "Rivera Alirio"},
			{ID: "assdro", Name: "Notararigo Alessandro"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
				"Fullmetal Alchemist",
				"Fate",
				"Hellsing",
				"Naruto",
				"Dragon Ball",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "john",
		Name:  "Doe John",
		Email: "john@example.com",
		Alive: false,

		Numbers: []int{1, 2, 3},

		Vehicle: Vehicle{
			ID:    2,
			Type:  "Car",
			Brand: "Jeep",
			Model: "Cherokee",
		},

		Family: []Person{
			{ID: "jane", Name: "Doe Jane"},
		},

		Doctype: "people",
	},

	{
		ID:    "luffy",
		Name:  "Monkey D. Luffy",
		Email: "luffy@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			30_000,
			100_000_000,
			300_000_000,
			400_000_000,
			500_000_000,
			1_500_000_000,
		},

		Vehicle: Vehicle{
			ID:    3,
			Type:  "Ship",
			Brand: "Franky's Ships",
			Model: "Thousands Sunny",
		},

		Family: []Person{
			{ID: "ace", Name: "Portgas D. Ace"},
			{ID: "sabo", Name: "Sabo"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "zoro",
		Name:  "Roronoa Zoro",
		Email: "zoro@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			60_000_000,
			120_000_000,
			320_000_000,
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "nami",
		Name:  "Nami",
		Email: "nami@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			16_000_000,
			66_000_000,
		},

		Family: []Person{
			{ID: "bell-mere", Name: "Bell-mere"},
			{ID: "nojiko", Name: "Nojiko"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "usopp",
		Name:  "Usopp",
		Email: "usopp@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			30_000_000,
			200_000_000,
		},

		Family: []Person{
			{ID: "yasopp", Name: "Yasopp"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "sanji",
		Name:  "Vinsmoke Sanji",
		Email: "sanji@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			77_000_000,
			177_000_000,
			330_000_000,
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "vivi",
		Name:  "Nefertari Vivi",
		Email: "vivi@mugiwaras.eb",
		Alive: true,

		Family: []Person{
			{ID: "cobra", Name: "Nefertari Cobra"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "karoo",
		Name:  "Karoo",
		Email: "robin@mugiwaras.eb",
		Alive: true,

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "chopper",
		Name:  "Tony Tony Chopper",
		Email: "chopper@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			50,
			100,
		},

		Family: []Person{
			{ID: "hiriluk", Name: "Hiriluk"},
			{ID: "doctorine", Name: "Kureha"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "robin",
		Name:  "Nico Robin",
		Email: "robin@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			79_000_000,
			80_000_000,
			130_000_000,
		},

		Family: []Person{
			{ID: "olvia", Name: "Nico Olvia"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "franky",
		Name:  "Cutty Flam",
		Email: "fanky@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			44_000_000,
			94_000_000,
		},

		Data: map[string]interface{}{
			"nickname": "Franky",
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "brook",
		Name:  "Brook",
		Email: "brook@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			33_000_000,
			83_000_000,
		},

		Family: []Person{
			{ID: "olvia", Name: "Nico Olvia"},
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},

	{
		ID:    "jinbe",
		Name:  "Jinbe",
		Email: "jinbe@mugiwaras.eb",
		Alive: true,

		Numbers: []int{
			76_000_000,
			250_000_000,
			438_000_000,
		},

		Data: map[string]interface{}{
			"anime": []string{
				"One Piece",
			},
		},

		Doctype: "people",
	},
}

Output:
Get -> ntrrg: Rivera Notararigo Miguel Angel
Find -> (Email:ntrrg@example.com): ["ntrrg"]
Find -> (Data.anime:"One Piece" Sorted:ID Limit:2): ["brook" "chopper"]
Prefix -> (nt): ["ntrrg"]
Delete -> ntrrg: Not found

Index

• Constants
• Variables
• func IsBadgerError(err error) bool
• func IsBleveError(err error) bool
• type DB
  • func Open(dir string) (*DB, error)
  • func OpenWith(opts Options) (db *DB, err error)
  • func (db *DB) Close() error
  • func (db *DB) NewTx(rw bool) *Tx
  • func (db *DB) ReloadIndex(f DecoderFunc) error
• type DecoderFunc
• type FindOptions
• type Options
  • func DefaultOptions(dir string) Options
  • func MemoryOptions() (Options, error)
• type SearchIndexOptions
• type Tx
  • func (tx *Tx) Commit() error
  • func (tx *Tx) Delete(key []byte) error
  • func (tx *Tx) Discard()
  • func (tx *Tx) Find(q string, opts *FindOptions) ([][]byte, error)
  • func (tx *Tx) Get(key []byte, v interface{}) error
  • func (tx *Tx) Prefix(prefix []byte) [][]byte
  • func (tx *Tx) Set(key []byte, v interface{}) (err error)

Examples

• Package
• DecoderFunc

Constants

const (
	InMemory       = ""
	DatabaseDir    = "database"
	SearchIndexDir = "search-index"
)

const (
	RW = true
	RO = false
)

Variables

var (
	ErrValMustBePointer = errors.New("can't encode data, must be a pointer")

	ErrKeyNotFound = badgerError(badger.ErrKeyNotFound)
	ErrTxnTooBig   = badgerError(badger.ErrTxnTooBig)
)

Functions

func IsBadgerError

func IsBadgerError(err error) bool

IsBadgerError returns true if the given error is from Badger.

func IsBleveError

func IsBleveError(err error) bool

IsBleveError returns true if the given error is from Bleve.

Types

type DB

type DB struct {
	// contains filtered or unexported fields
}

DB is a database object which provides database management methods, for data management see Tx.

func Open

func Open(dir string) (*DB, error)

Open initializes a database in the given directory.

func OpenWith

func OpenWith(opts Options) (db *DB, err error)

OpenWith initializes a database with the given options.

func (*DB) Close

func (db *DB) Close() error

Close terminates the database.

func (*DB) NewTx

func (db *DB) NewTx(rw bool) *Tx

NewTx creates a database transaction. If rw is false, the new transaction will be read-only.

func (*DB) ReloadIndex

func (db *DB) ReloadIndex(f DecoderFunc) error

ReloadIndex recreates the search index, it takes a decoder function as argument, this is necessary since it is not possible to decode one type into another.

type DecoderFunc

type DecoderFunc func(tx *Tx, key []byte) (interface{}, error)

Example

package main

import (
	"bytes"
	"fmt"

	"nt.web.ve/go/sdb/pkg/sdb"
)

func main() {
	db, err := sdb.Open(sdb.InMemory)
	if err != nil {
		panic(err)
	}

	defer db.Close()

	f := func(tx *sdb.Tx, key []byte) (interface{}, error) {
		switch {
		case bytes.HasPrefix(key, []byte("strings-")):
			var v string

			if errGet := tx.Get(key, &v); err != nil {
				return nil, errGet
			}

			return v, nil
		case bytes.HasPrefix(key, []byte("numbers-")):
			var v int

			if errGet := tx.Get(key, &v); err != nil {
				return nil, errGet
			}

			return v, nil
		}

		return nil, fmt.Errorf("unknown type")
	}

	if errReload := db.ReloadIndex(f); err != nil {
		panic(errReload)
	}

}

type FindOptions

type FindOptions struct {
	// A comma-separated list of fields used for sorting, any field prefixed by a
	// hyphen (-) will be reverse ordered.
	Sort string

	// Amount of keys to be retrieved. 0 means no limit.
	Limit int

	// Amount of keys per page. 0 means no pagination.
	PageSize int

	// Page number starting from 0. If PageSize is 5, and Page is 2, it will
	// retrieve the 10-14 keys.
	Page int

	// A function that filters the retrieved keys. Returning false means the key
	// must be omitted.
	Filter func(tx *Tx, key []byte) (ok bool, err error)
}

FindOptions controls the behavior of Tx.Find.

type Options

type Options struct {
	// Database location.
	Dir string

	Badger badger.Options
	Bleve  SearchIndexOptions

	BufferPoolSize     int  // Amount of buffers.
	BufferPoolMaxBytes int  // Bytes limit per buffer.
	BufferPoolFill     bool // Fill up the pool at DB creation.

	Logger *log.Logger
}

Options are parameters for initializing a database.

func DefaultOptions

func DefaultOptions(dir string) Options

DefaultOptions returns commonly used options for creating a database.

func MemoryOptions

func MemoryOptions() (Options, error)

MemoryOptions returns options tweaked for running a DB instance in memory.

type SearchIndexOptions

type SearchIndexOptions struct {
	Dir          string
	DoctypeField string
	DocMappings  map[string]*mapping.DocumentMapping
}

SearchIndexOptions are parameters for initializing the search index.

type Tx

type Tx struct {
	// contains filtered or unexported fields
}

Tx is a transaction object which provides data management methods. The search index doesn't support transactions yet, so indexing operations just take effect after committing the transaction.

func (*Tx) Commit

func (tx *Tx) Commit() error

Commit writes the transaction operations to the database. If a Bleve error is returned, the search index should be reloaded (see DB.ReloadIndex), keep the amount of operations per transaction low to avoid this.

func (*Tx) Delete

func (tx *Tx) Delete(key []byte) error

Delete deletes the given key. This operation happens in memory, it will be written to the database once Commit is called.

func (*Tx) Discard

func (tx *Tx) Discard()

Discard drops all the pending modifications and set the transactions as discarded.

func (*Tx) Find

func (tx *Tx) Find(q string, opts *FindOptions) ([][]byte, error)

Find fetches the keys from the values that satisfies the given constraints. See http://blevesearch.com/docs/Query-String-Query/ for more info about the the query language syntax. See also FindOptions.

func (*Tx) Get

func (tx *Tx) Get(key []byte, v interface{}) error

Get reads the value from the given key and decodes it into v. v must be a pointer.

func (*Tx) Prefix

func (tx *Tx) Prefix(prefix []byte) [][]byte

Prefix fetches all the keys from the database with the given prefix.

func (*Tx) Set

func (tx *Tx) Set(key []byte, v interface{}) (err error)

Set set v as value of the given key. This operation happens in memory, it will be written to the database once Commit is called. v must be a pointer.