bytengine

README

Bytengine

About

Bytengine is a scalable content repository built with Go. Its API is accessible from any Http client library so you can start coding in your favorite language!

Bytengine stores your JSON data and digital assets in a pseudo hierarchical file system which you query using it's inbuilt SQL like language.

Some of the server's features are:

• JSON data management
• Digital assets management
• HTTP based API
• Bytengine Query language (BQL)
• Pluggable data storage backends (currently supports Mongodb, Diskv, Redis)
• Command line interface bshell

Installation

Current Build Prerequisites:

• Mongodb
• Redis

You can download Bytengine binaries for:

• Linux amd64
• Mac OS X 10.6/10.7 amd64
• Raspberry Pi

Extract downloaded file, 'cd' into directory and run:

    ./bytengine createadmin -u="admin" -p="yourpassword"
    ./bytengine run

Development

Bytengine is developed on OS X so you should adapt the following instructions to your Os/Distro (Only tested on OS X, Ubuntu Linux, Raspbian)

Current Build Prerequisites:

• Mongodb
• Redis
• Go

1. Get Bytengine go get -d github.com/johnwilson/bytengine/cmd/bytengine

2. cd $GOPATH/src/github.com/johnwilson/bytengine/cmd/bytengine

3. Build Bytengine go build

4. Rename config.json.sample to config.json

5. Running Bytengine

	./bytengine createadmin -u="admin" -p="yourpassword"
	./bytengine run

Quick Tutorial

Using Python + Requests

    >>> import requests
    >>> url = "http://localhost:8500/bfs/token"
    >>> data = {"username":"user","password":"password"}
    >>> r = requests.post(url, data=data)
    >>> j = r.json()
    >>> print j["status"]
    ok
    >>> token = j["data"]
    >>> cmd = 'server.newdb "test"; server.listdb;'  # issue two commands
    >>> url = "http://localhost:8500/bfs/query"
    >>> data = {"token":token,"query":cmd}
    >>> r = requests.post(url, data=data)
    >>> j = r.json()
    >>> print j["status"]
    ok
    >>> print j["data"][-1]  # get last result
    [u'test']

Using Bytengine Shell bshell

Login:

    bshell run -u=user -p=password

Enter commands:

    bql> server.newdb "test"; server.listdb;
    {
      "data": [
        true,
        [
          "test"
        ]
      ],
      "status": "ok"
    }
    bql> \s lastresult().status
    ok

Some handy links

Documentation - Bytengine Docs

Twitter - Follow Bytengine

Hemi v6 - Deploy to Heroku

Thanks

Go project

golang-nuts community

Third-party libraries/framework developers

Documentation

Index

• Variables
• func RegisterAuthentication(name string, plugin Authentication)
• func RegisterByteStore(name string, plugin ByteStore)
• func RegisterCommandHandler(name string, fn CommandHandler)
• func RegisterDataFilter(name string, fn DataFilter)
• func RegisterFileSystem(name string, plugin FileSystem)
• func RegisterParser(name string, plugin Parser)
• func RegisterStateStore(name string, plugin StateStore)
• type Authentication
  • func NewAuthentication(pluginName, config string) (plugin Authentication, err error)
• type ByteStore
  • func NewByteStore(pluginName, config string) (plugin ByteStore, err error)
• type Command
  • func (c Command) String() string
• type CommandHandler
• type Config
• type ConfigData
• type DataFilter
• type Engine
  • func NewEngine() *Engine
  • func (eng *Engine) CreateAdminUser(usr, pw string) error
  • func (eng *Engine) ExecuteCommand(token string, cmd Command) (interface{}, error)
  • func (eng *Engine) ExecuteScript(token, script string) (interface{}, error)
  • func (eng *Engine) Start(b []byte) error
• type FileSystem
  • func NewFileSystem(pluginName, config string, b *ByteStore) (plugin FileSystem, err error)
• type Parser
  • func NewParser(pluginName, config string) (plugin Parser, err error)
• type StateStore
  • func NewStateStore(pluginName, config string) (plugin StateStore, err error)
• type User

Variables

var (
	// to be expanded
	MimeList = map[string]string{
		".js":  "text/javascript",
		".css": "text/css",
	}
)

Functions

func RegisterAuthentication

func RegisterAuthentication(name string, plugin Authentication)

func RegisterByteStore

func RegisterByteStore(name string, plugin ByteStore)

func RegisterCommandHandler

func RegisterCommandHandler(name string, fn CommandHandler)

func RegisterDataFilter

func RegisterDataFilter(name string, fn DataFilter)

func RegisterFileSystem

func RegisterFileSystem(name string, plugin FileSystem)

func RegisterParser

func RegisterParser(name string, plugin Parser)

func RegisterStateStore

func RegisterStateStore(name string, plugin StateStore)

Types

type Authentication

type Authentication interface {
	Start(config string) error
	ClearAll() error
	Authenticate(usr, pw string) bool
	NewUser(usr, pw string, root bool) error
	ChangeUserPassword(usr, pw string) error
	ChangeUserStatus(usr string, isactive bool) error
	ListUser(rgx string) ([]string, error)
	ChangeUserDbAccess(usr, db string, grant bool) error
	HasDbAccess(usr, db string) bool
	RemoveUser(usr string) error
	UserInfo(u string) (*User, error)
}

func NewAuthentication

func NewAuthentication(pluginName, config string) (plugin Authentication, err error)

type ByteStore

type ByteStore interface {
	Start(config string) error
	Add(db string, file *os.File) (map[string]interface{}, error)
	Update(db, id string, file *os.File) (map[string]interface{}, error)
	Delete(db, id string) error
	Read(db, filename string, file io.Writer) error
	DropDatabase(db string) error
}

func NewByteStore

func NewByteStore(pluginName, config string) (plugin ByteStore, err error)

type Command

type Command struct {
	Name     string
	Database string
	Args     map[string]interface{}
	Options  map[string]interface{}
	Filter   string // name of filter to apply to result
	IsAdmin  bool   // is it an admin command
}

func (Command) String

func (c Command) String() string

type CommandHandler

type CommandHandler func(cmd Command, user *User, eng *Engine) (interface{}, error)

type Config

type Config struct {
	Authentication struct{ Plugin string }
	ByteStore      struct{ Plugin string }
	FileSystem     struct{ Plugin string }
	StateStore     struct{ Plugin string }
	DataFilter     struct{ Plugin string }
	Parser         struct{ Plugin string }
}

type ConfigData

type ConfigData struct {
	Authentication json.RawMessage
	ByteStore      json.RawMessage
	FileSystem     json.RawMessage
	StateStore     json.RawMessage
	DataFilter     json.RawMessage
	Parser         json.RawMessage
}

type DataFilter

type DataFilter func(r interface{}, eng *Engine) (interface{}, error)

type Engine

type Engine struct {
	Authentication Authentication
	FileSystem     FileSystem
	ByteStore      ByteStore
	StateStore     StateStore
	Parser         Parser
}

func NewEngine

func NewEngine() *Engine

func (*Engine) CreateAdminUser

func (eng *Engine) CreateAdminUser(usr, pw string) error

func (*Engine) ExecuteCommand

func (eng *Engine) ExecuteCommand(token string, cmd Command) (interface{}, error)

command is sent directly for execution

func (*Engine) ExecuteScript

func (eng *Engine) ExecuteScript(token, script string) (interface{}, error)

script is parsed into commands before execution

func (*Engine) Start

func (eng *Engine) Start(b []byte) error

start engine and configure plugins

type FileSystem

type FileSystem interface {
	Start(config string, b *ByteStore) error
	ClearAll() ([]string, error)
	ListDatabase(filter string) ([]string, error)
	CreateDatabase(db string) error
	DropDatabase(db string) error
	NewDir(p, db string) error
	NewFile(p, db string, jsondata map[string]interface{}) error
	ListDir(p, filter, db string) (map[string][]string, error)
	ReadJson(p, db string, fields []string) (interface{}, error)
	Delete(p, db string) error
	Rename(p, newname, db string) error
	Move(from, to, db string) error
	Copy(from, to, db string) error
	Info(p, db string) (map[string]interface{}, error)
	FileAccess(p, db string, protect bool) error
	SetCounter(counter, action string, value int64, db string) (int64, error)
	ListCounter(filter, db string) (map[string]int64, error)
	WriteBytes(p, ap, db string) (int64, error)
	ReadBytes(fp, db string) (string, error)
	DirectAccess(fp, db, layer string) (map[string]interface{}, string, error)
	DeleteBytes(p, db string) error
	UpdateJson(p, db string, j map[string]interface{}) error
	BQLSearch(db string, query map[string]interface{}) (interface{}, error)
	BQLSet(db string, query map[string]interface{}) (int, error)
	BQLUnset(db string, query map[string]interface{}) (int, error)
}

func NewFileSystem

func NewFileSystem(pluginName, config string, b *ByteStore) (plugin FileSystem, err error)

type Parser

type Parser interface {
	Parse(s string) (c []Command, err error)
}

func NewParser

func NewParser(pluginName, config string) (plugin Parser, err error)

type StateStore

type StateStore interface {
	TokenSet(token, user string, timeout int64) error
	TokenGet(token string) (string, error)
	CacheSet(id, value string, timeout int64) error
	CacheGet(id string) (string, error)
	ClearAll() error
	Start(config string) error
}

Manages authentication tokens, upload tickets and caching

func NewStateStore

func NewStateStore(pluginName, config string) (plugin StateStore, err error)

type User

type User struct {
	Username  string   `json:"username"`
	Active    bool     `json:"active"`
	Databases []string `json:"databases"`
	Root      bool     `json:"root"`
}