bolt

package module
v0.2.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 18, 2017 License: MIT Imports: 23 Imported by: 1

README

Golang Neo4J Bolt Driver

Build Status Tested against Golang 1.4.3 and up

Implements the Neo4J Bolt Protocol specification: As of the time of writing this, the current version is v3.1.0-M02

go get github.com/axiomzen/golang-neo4j-bolt-driver

Features

  • Neo4j Bolt low-level binary protocol support
  • Message Pipelining for high concurrency
  • Connection Pooling
  • TLS support
  • Compatible with sql.driver

Usage

Please see the statement tests or the conn tests for A LOT of examples of usage

Examples
Quick n’ Dirty
func quickNDirty() {
	driver := bolt.NewDriver()
	conn, _ := driver.OpenNeo("bolt://localhost:7687")
	defer conn.Close()

	// Start by creating a node
	result, _ := conn.ExecNeo("CREATE (n:NODE {foo: {foo}, bar: {bar}})", map[string]interface{}{"foo": 1, "bar": 2.2})
	numResult, _ := result.RowsAffected()
	fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 1

	// Lets get the node
	data, rowsMetadata, _, _ := conn.QueryNeoAll("MATCH (n:NODE) RETURN n.foo, n.bar", nil)
	fmt.Printf("COLUMNS: %#v\n", rowsMetadata["fields"].([]interface{}))  // COLUMNS: n.foo,n.bar
	fmt.Printf("FIELDS: %d %f\n", data[0][0].(int64), data[0][1].(float64)) // FIELDS: 1 2.2

	// oh cool, that worked. lets blast this baby and tell it to run a bunch of statements
	// in neo concurrently with a pipeline
	results, _ := conn.ExecPipeline([]string{
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
	}, nil, nil, nil, nil, nil, nil)
	for _, result := range results {
		numResult, _ := result.RowsAffected()
		fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 2 (per each iteration)
	}

	data, _, _, _ = conn.QueryNeoAll("MATCH (n:NODE)-[:REL]->(m) RETURN m", nil)
	for _, row := range data {
		fmt.Printf("NODE: %#v\n", row[0].(graph.Node)) // Prints all nodes
	}

	result, _ = conn.ExecNeo(`MATCH (n) DETACH DELETE n`, nil)
	numResult, _ = result.RowsAffected()
	fmt.Printf("Rows Deleted: %d", numResult) // Rows Deleted: 13
}
Slow n' Clean
func slowNClean() {
	driver := bolt.NewDriver()
	conn, err := driver.OpenNeo("bolt://localhost:7687")
	if err != nil {
		panic(err)
	}
	defer conn.Close()

	// Here we prepare a new statement. This gives us the flexibility to
	// cancel that statement without any request sent to Neo
	stmt, err := conn.PrepareNeo("CREATE (n:NODE {foo: {foo}, bar: {bar}})")
	if err != nil {
		panic(err)
	}

	// Executing a statement just returns summary information
	result, err := stmt.ExecNeo(map[string]interface{}{"foo": 1, "bar": 2.2})
	if err != nil {
		panic(err)
	}
	numResult, err := result.RowsAffected()
	if err != nil {
		panic(err)
	}
	fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 1

	// Closing the statment will also close the rows
	stmt.Close()

	// Lets get the node. Once again I can cancel this with no penalty
	stmt, err = conn.PrepareNeo("MATCH (n:NODE) RETURN n.foo, n.bar")
	if err != nil {
		panic(err)
	}

	// Even once I get the rows, if I do not consume them and close the
	// rows, Neo will discard and not send the data
	rows, err := stmt.QueryNeo(nil)
	if err != nil {
		panic(err)
	}

	// This interface allows you to consume rows one-by-one, as they
	// come off the bolt stream. This is more efficient especially
	// if you're only looking for a particular row/set of rows, as
	// you don't need to load up the entire dataset into memory
	data, _, err := rows.NextNeo()
	if err != nil {
		panic(err)
	}

	// This query only returns 1 row, so once it's done, it will return
	// the metadata associated with the query completion, along with
	// io.EOF as the error
	_, _, err = rows.NextNeo()
	if err != io.EOF {
		panic(err)
	}
	fmt.Printf("COLUMNS: %#v\n", rows.Metadata()["fields"].([]interface{})) // COLUMNS: n.foo,n.bar
	fmt.Printf("FIELDS: %d %f\n", data[0].(int64), data[1].(float64))       // FIELDS: 1 2.2

	stmt.Close()

	// Here we prepare a new pipeline statement for running multiple
	// queries concurrently
	pipeline, err := conn.PreparePipeline(
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
	)
	if err != nil {
		panic(err)
	}

	pipelineResults, err := pipeline.ExecPipeline(nil, nil, nil, nil, nil, nil)
	if err != nil {
		panic(err)
	}

	for _, result := range pipelineResults {
		numResult, _ := result.RowsAffected()
		fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 2 (per each iteration)
	}

	err = pipeline.Close()
	if err != nil {
		panic(err)
	}

	stmt, err = conn.PrepareNeo("MATCH path=(n:NODE)-[:REL]->(m) RETURN path")
	if err != nil {
		panic(err)
	}

	rows, err = stmt.QueryNeo(nil)
	if err != nil {
		panic(err)
	}

	// Here we loop through the rows until we get the metadata object
	// back, meaning the row stream has been fully consumed
	for err == nil {
		var row []interface{}
		row, _, err = rows.NextNeo()
		if err != nil && err != io.EOF {
			panic(err)
		} else if err != io.EOF {
			fmt.Printf("PATH: %#v\n", row[0].(graph.Path)) // Prints all paths
		}
	}

	stmt.Close()

	result, _ = conn.ExecNeo(`MATCH (n) DETACH DELETE n`, nil)
	fmt.Println(result)
	numResult, _ = result.RowsAffected()
	fmt.Printf("Rows Deleted: %d", numResult) // Rows Deleted: 13
}

API

There is much more detailed information in the godoc

This implementation attempts to follow the best practices as per the Bolt specification, but also implements compatibility with Golang's sql.driver interface.

As such, these interfaces closely match the sql.driver interfaces, but they also provide Neo4j Bolt specific functionality in addition to the sql.driver interface.

It is recommended that you use the Neo4j Bolt-specific interfaces if possible. The implementation is more efficient and can more closely support the Neo4j Bolt feature set.

The URL format is: bolt://(user):(password)@(host):(port) Schema must be bolt. User and password is only necessary if you are authenticating.

Connection pooling is provided out of the box with the NewDriverPool method. A bunch of options (use DefaultPoolOptions for defaults)

You can get logs from the driver by setting the log level using the log packages SetLevel.

Dev Quickstart

# Put in git hooks
ln -s ../../scripts/pre-commit .git/hooks/pre-commit
ln -s ../../scripts/pre-push .git/hooks/pre-push

# No special build steps necessary
go build

# Testing with log info and a local bolt DB, getting coverage output
BOLT_DRIVER_LOG=info NEO4J_BOLT=bolt://localhost:7687 go test -coverprofile=./tmp/cover.out -coverpkg=./... -v -race && go tool cover -html=./tmp/cover.out

# Testing with trace output for debugging
BOLT_DRIVER_LOG=trace NEO4J_BOLT=bolt://localhost:7687 go test -v -race

# Testing with running recorder to record tests for CI
BOLT_DRIVER_LOG=trace NEO4J_BOLT=bolt://localhost:7687 RECORD_OUTPUT=1 go test -v -race

The tests are written in an integration testing style. Most of them are in the statement tests, but should be made more granular in the future.

In order to get CI, I made a recorder mechanism so you don't need to run neo4j alongside the tests in the CI server. You run the tests locally against a neo4j instance with the RECORD_OUTPUT=1 environment variable, it generates the recordings in the ./recordings folder. This is necessary if the tests have changed, or if the internals have significantly changed. Installing the git hooks will run the tests automatically on push. If there are updated tests, you will need to re-run the recorder to add them and push them as well.

You need access to a running Neo4J database to develop for this project, so that you can run the tests to generate the recordings.

Easiest way is with docker:

docker run --publish=7474:7474 --publish=7687:7687 --env=NEO4J_AUTH=none --volume=$HOME/neo4j/data:/data neo4j

TODO

  • Cypher Parser to implement NumInput and pre-flight checking
  • More Tests
  • Benchmark Tests

Documentation

Overview

package bolt implements a driver for the Neo4J Bolt Protocol.

The driver is compatible with Golang's sql.driver interface, but aims to implement a more complete featureset in line with what Neo4J and Bolt provides.

As such, there are multiple interfaces the user can choose from. It's highly recommended that the user use the Neo4J-specific interfaces as they are more flexible and efficient than the provided sql.driver compatible methods.

The interface tries to be consistent throughout. The sql.driver interfaces are standard, but the Neo4J-specific ones contain a naming convention of either "Neo" or "Pipeline".

The "Neo" ones are the basic interfaces for making queries to Neo4j and it's expected that these would be used the most.

The "Pipeline" ones are to support Bolt's pipelining features. Pipelines allow the user to send Neo4j many queries at once and have them executed by the database concurrently. This is useful if you have a bunch of queries that aren't necessarily dependant on one another, and you want to get better performance. The internal APIs will also pipeline statements where it is able to reliably do so, but by manually using the pipelining feature you can maximize your throughput.

The API provides connection pooling using the `NewDriverPool` method. This allows you to pass it the maximum number of open connections to be used in the pool. Once this limit is hit, any new clients will have to wait for a connection to become available again.

The sql driver is registered as "neo4j-bolt". The sql.driver interface is much more limited than what bolt and neo4j supports. In some cases, concessions were made in order to make that interface work with the neo4j way of doing things. The main instance of this is the marshalling of objects to/from the sql.driver.Value interface. In order to support object types that aren't supported by this interface, the internal encoding package is used to marshal these objects to byte strings. This ultimately makes for a less efficient and more 'clunky' implementation. A glaring instance of this is passing parameters. Neo4j expects named parameters but the driver interface can only really support positional parameters. To get around this, the user must create a map[string]interface{} of their parameters and marshal it to a driver.Value using the encoding.Marshal function. Similarly, the user must unmarshal data returned from the queries using the encoding.Unmarshal function, then use type assertions to retrieve the proper type.

In most cases the driver will return the data from neo as the proper go-specific types. For integers they always come back as int64 and floats always come back as float64. This is for the convenience of the user and acts similarly to go's JSON interface. This prevents the user from having to use reflection to get these values. Internally, the types are always transmitted over the wire with as few bytes as possible.

There are also cases where no go-specific type matches the returned values, such as when you query for a node, relationship, or path. The driver exposes specific structs which represent this data in the 'structures.graph' package. There are 4 types - Node, Relationship, UnboundRelationship, and Path. The driver returns interface{} objects which must have their types properly asserted to get the data out.

There are some limitations to the types of collections the driver supports. Specifically, maps should always be of type map[string]interface{} and lists should always be of type []interface{}. It doesn't seem that the Bolt protocol supports uint64 either, so the biggest number it can send right now is the int64 max.

The URL format is: `bolt://(user):(password)@(host):(port)` Schema must be `bolt`. User and password is only necessary if you are authenticating. TLS is supported by using query parameters on the connection string, like so: `bolt://host:port?tls=true&tls_no_verify=false`

The supported query params are:

* timeout - the number of seconds to set the connection timeout to. Defaults to 60 seconds. * tls - Set to 'true' or '1' if you want to use TLS encryption * tls_no_verify - Set to 'true' or '1' if you want to accept any server certificate (for testing, not secure) * tls_ca_cert_file - path to a custom ca cert for a self-signed TLS cert * tls_cert_file - path to a cert file for this client (need to verify this is processed by Neo4j) * tls_key_file - path to a key file for this client (need to verify this is processed by Neo4j)

Errors returned from the API support wrapping, so if you receive an error from the library, it might be wrapping other errors. You can get the innermost error by using the `InnerMost` method. Failure messages from Neo4J are reported, along with their metadata, as an error. In order to get the failure message metadata from a wrapped error, you can do so by calling `err.(*errors.Error).InnerMost().(messages.FailureMessage).Metadata`

If there is an error with the database connection, you should get a sql/driver ErrBadConn as per the best practice recommendations of the Golang SQL Driver. However, this error may be wrapped, so you might have to call `InnerMost` to get it, as specified above.

Index

Constants

This section is empty.

Variables

View Source 
var (

	// Version is the current version of this driver
	Version = "1.0"
	// ClientID is the id of this client
	ClientID = "GolangNeo4jBolt/" + Version
)
View Source 
var ErrClosed = errors.New("connection is closed")

ErrClosed is when you attempt to use a closed connection

View Source 
var ErrStmtAlreadyClosed = errors.New("Neo4j Bolt statement already closed")

ErrStmtAlreadyClosed is when you are trying to exec on a statement that is alreay closed

View Source 
var ErrTxClose = errors.New("Transaction already closed")

ErrTxClose is for when you try and close an already closed tx

Functions

func TLSHelper 

func TLSHelper(skipVerify bool, caCertFile, certFile, keyFile string) (*tls.Config, error)

TLSHelper creates a tls.Config

Types

type Conn 

type Conn interface {
	interpool.Conn
	// PrepareNeo prepares a neo4j specific statement
	PrepareNeo(query string) (Stmt, error)
	// PreparePipeline prepares a neo4j specific pipeline statement
	// Useful for running multiple queries at the same time
	PreparePipeline(query ...string) (PipelineStmt, error)
	// QueryNeo queries using the neo4j-specific interface
	QueryNeo(query string, params map[string]interface{}) (Rows, error)
	// QueryNeoAll queries using the neo4j-specific interface and returns all row data and output metadata
	QueryNeoAll(query string, params map[string]interface{}) ([][]interface{}, map[string]interface{}, map[string]interface{}, error)
	// QueryPipeline queries using the neo4j-specific interface
	// pipelining multiple statements
	QueryPipeline(query []string, params ...map[string]interface{}) (PipelineRows, error)
	// ExecNeo executes a query using the neo4j-specific interface
	ExecNeo(query string, params map[string]interface{}) (Result, error)
	// ExecPipeline executes a query using the neo4j-specific interface
	// pipelining multiple statements
	ExecPipeline(query []string, params ...map[string]interface{}) ([]Result, error)
	// Begin starts a new transaction
	Begin() (driver.Tx, error)
	// SetChunkSize is used to set the max chunk size of the
	// bytes to send to Neo4j at once
	SetChunkSize(uint16)
}

Conn represents a connection to Neo4J

Implements a neo-friendly interface. Some of the features of this interface implement neo-specific features unavailable in the sql/driver compatible interface

Conn objects, and any prepared statements/transactions within ARE NOT THREAD SAFE. If you want to use multipe go routines with these objects, you should use a driver to create a new conn for each routine.

type Driver 

type Driver interface {
	// Open opens a sql.driver compatible connection. Used internally
	// by the go sql interface
	Open(constr string) (driver.Conn, error)
	// OpenNeo opens a Neo-specific connection. This should be used
	// directly when not using the golang sql interface
	OpenNeo() (Conn, error)
	// NewConnPool creates a new connection pool instead
	NewConnPool(ops *PoolOptions) DriverPool
}

Driver is a driver allowing connection to Neo4j The driver allows you to open a new connection to Neo4j

Implements sql/driver, but also includes its own more neo-friendly interface. Some of the features of this interface implement neo-specific features unavailable in the sql/driver compatible interface

Driver objects should be THREAD SAFE, so you can use them to open connections in multiple threads. The connection objects themselves, and any prepared statements/transactions within ARE NOT THREAD SAFE.

func NewDriver 

func NewDriver() Driver

NewDriver creates a new Driver object with defaults

func NewDriverWithOptions 

func NewDriverWithOptions(options *DriverOptions) Driver

NewDriverWithOptions creates a new Driver object with the given options

type DriverOptions 

type DriverOptions struct {
	// Dial timeout for establishing new connections.
	// Default is 5 seconds.
	DialTimeout time.Duration
	// Timeout for socket reads. If reached, commands will fail
	// with a timeout instead of blocking.
	ReadTimeout time.Duration
	// Timeout for socket writes. If reached, commands will fail
	// with a timeout instead of blocking.
	WriteTimeout time.Duration
	// Addr is the connection string
	Addr string
	// TLSNoverify allows you to skip tls verification
	// replaced by TLSConfig.InsecureSkipVerify
	//TLSNoVerify bool
	// TLSConfig is the tls configuration (nil by default)
	TLSConfig *tls.Config
	// PoolOptions are the options for the connection pool
	//PoolOptions *PoolOptions
	// ChunkSize is im not sure
	ChunkSize uint16
}

DriverOptions are the options for the driver

func DefaultDriverOptions 

func DefaultDriverOptions() *DriverOptions

DefaultDriverOptions returns the default options

type DriverPool 

type DriverPool interface {
	io.Closer
	// Get gets a connection from the pool
	Get() (Conn, error)
	// Put puts the connection back into the pool
	// for later reuse
	Put(Conn) error
	// Closed lets you know if we are closed or not
	Closed() bool
	// Len gives the size of the pool
	Len() int
	// FreeLen gives the number of free connections
	FreeLen() int
}

DriverPool is a driver allowing connection to Neo4j with support for connection pooling The driver allows you to open a new connection to Neo4j

DriverPool objects should be THREAD SAFE, so you can use them to open connections in multiple threads. The connection objects themselves, and any prepared statements/transactions within ARE NOT THREAD SAFE.

type Event 

type Event struct {
	Timestamp int64 `json:"-"`
	Event     []byte
	IsWrite   bool
	Completed bool
	Error     error
}

Event represents a single recording (read or write) event in the recorder

type PipelineRows 

type PipelineRows interface {
	// Columns Gets the names of the columns in the returned dataset
	Columns() []string
	// Metadata Gets all of the metadata returned from Neo on query start
	Metadata() map[string]interface{}
	// Close the rows, flushing any existing datastream
	Close() error
	// NextPipeline gets the next row result
	// When the rows are completed, returns the success metadata and the next
	// set of rows.
	// When all rows are completed, returns io.EOF
	NextPipeline() ([]interface{}, map[string]interface{}, PipelineRows, error)
}

PipelineRows represents results of a set of rows from the DB when running a pipeline statement.

Row objects ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type PipelineStmt 

type PipelineStmt interface {
	// Close Closes the statement. See sql/driver.Stmt.
	Close() error
	// ExecPipeline executes a set of queries that returns no rows.
	ExecPipeline(params ...map[string]interface{}) ([]Result, error)
	// QueryPipeline executes a set of queries that return data.
	// Implements a Neo-friendly alternative to sql/driver.
	QueryPipeline(params ...map[string]interface{}) (PipelineRows, error)
}

PipelineStmt represents a set of statements to run against the database

PipelineStmt objects, and any rows prepared within ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type PoolOptions 

type PoolOptions struct {
	PoolSize           int
	PoolTimeout        time.Duration
	IdleTimeout        time.Duration
	IdleCheckFrequency time.Duration
	MaxAge             time.Duration
}

PoolOptions are the options for the connection pool

func DefaultPoolOptions 

func DefaultPoolOptions() *PoolOptions

DefaultPoolOptions returns the default connection pool options

type Result 

type Result interface {
	// LastInsertId Always returns -1. This is necessary
	// to meet the sql.driver interface
	LastInsertId() (int64, error)
	// RowsAffected returns the number of rows affected
	// This doesn't currently support updates, only
	// inserts/deletions
	RowsAffected() (int64, error)
	// Metadata returns the metadata response from neo4j
	Metadata() map[string]interface{}
}

Result represents a result from a query that returns no data

type Rows 

type Rows interface {
	// Columns Gets the names of the columns in the returned dataset
	Columns() []string
	// Metadata Gets all of the metadata returned from Neo on query start
	Metadata() map[string]interface{}
	// Close the rows, flushing any existing datastream
	Close() error
	// NextNeo gets the next row result
	// When the rows are completed, returns the success metadata
	// and io.EOF
	NextNeo() ([]interface{}, map[string]interface{}, error)
	// All gets all of the results from the row set. It's recommended to use NextNeo when
	// there are a lot of rows
	All() ([][]interface{}, map[string]interface{}, error)
}

Rows represents results of rows from the DB

Row objects ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type Stmt 

type Stmt interface {
	// Close Closes the statement. See sql/driver.Stmt.
	Close() error
	// ExecNeo executes a query that returns no rows. Implements a Neo-friendly alternative to sql/driver.
	ExecNeo(params map[string]interface{}) (Result, error)
	// QueryNeo executes a query that returns data. Implements a Neo-friendly alternative to sql/driver.
	QueryNeo(params map[string]interface{}) (Rows, error)
}

Stmt represents a statement to run against the database

Stmt objects, and any rows prepared within ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type Tx 

type Tx interface {
	// Commit commits the transaction
	Commit() error
	// Rollback rolls back the transaction
	Rollback() error
}

Tx represents a transaction

type URLParse 

type URLParse struct {
	//Scheme      string
	URL         *url.URL
	User        string
	Password    string
	DialTimeout time.Duration
	UseTLS      bool
	CertFile    string
	KeyFile     string
	CaCertFile  string
	TLSNoVerify bool
}

URLParse is the result from parsing the url

func (*URLParse) GetTLSConfig 

func (p *URLParse) GetTLSConfig() (*tls.Config, error)

GetTLSConfig is a helper function to extract the tls config

func (*URLParse) ParseURL 

func (p *URLParse) ParseURL(addr string) error

ParseURL parses a bolt url

Source Files

View all Source files

Directories

Path Synopsis
Package encoding is used to encode/decode data going to/from the bolt protocol.
 Package encoding is used to encode/decode data going to/from the bolt protocol.
Package errors contains the errors used by the bolt driver.
 Package errors contains the errors used by the bolt driver.
examples
playground
quick_n_dirty
slow_n_clean
Package log implements the logging for the bolt driver There are 3 logging levels - trace, info and error.
 Package log implements the logging for the bolt driver There are 3 logging levels - trace, info and error.
Package structures contains various structures which are used by the Bolt protocol
 Package structures contains various structures which are used by the Bolt protocol
graph
Package graph contains structs that can be returned from the Neo4j Graph
 Package graph contains structs that can be returned from the Neo4j Graph
messages
Package messages contains structs that represent the messages that get sent using the Bolt protocol
 Package messages contains structs that represent the messages that get sent using the Bolt protocol

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.