golangNeo4jBoltDriver

README

Golang Neo4J Bolt Driver

ANNOUNCEMENT: I must apologize to the community for not being more responsive. Because of personal life events I am really not able to properly maintain this codebase. Certain other events lead me to believe an official Neo4J Golang driver was to be released soon, but it seems like that may not necessarily be the case. Since I am unable to properly maintain this codebase at this juncture, at this point I think it makes sense to open up this repo to collaborators who are interested in helping with maintenance. Please feel free to email me directly if you're interested.

Implements the Neo4J Bolt Protocol specification: As of the time of writing this, the current version is v3.1.0-M02

go get github.com/johnnadratowski/golang-neo4j-bolt-driver

Features

• Neo4j Bolt low-level binary protocol support
• Message Pipelining for high concurrency
• Connection Pooling
• TLS support
• Compatible with sql.driver

Usage

Please see the statement tests or the conn tests for A LOT of examples of usage

Examples

Quick n’ Dirty

func quickNDirty() {
	driver := bolt.NewDriver()
	conn, _ := driver.OpenNeo("bolt://username:password@localhost:7687")
	defer conn.Close()

	// Start by creating a node
	result, _ := conn.ExecNeo("CREATE (n:NODE {foo: {foo}, bar: {bar}})", map[string]interface{}{"foo": 1, "bar": 2.2})
	numResult, _ := result.RowsAffected()
	fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 1

	// Lets get the node
	data, rowsMetadata, _, _ := conn.QueryNeoAll("MATCH (n:NODE) RETURN n.foo, n.bar", nil)
	fmt.Printf("COLUMNS: %#v\n", rowsMetadata["fields"].([]interface{}))  // COLUMNS: n.foo,n.bar
	fmt.Printf("FIELDS: %d %f\n", data[0][0].(int64), data[0][1].(float64)) // FIELDS: 1 2.2

	// oh cool, that worked. lets blast this baby and tell it to run a bunch of statements
	// in neo concurrently with a pipeline
	results, _ := conn.ExecPipeline([]string{
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
	}, nil, nil, nil, nil, nil, nil)
	for _, result := range results {
		numResult, _ := result.RowsAffected()
		fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 2 (per each iteration)
	}

	data, _, _, _ = conn.QueryNeoAll("MATCH (n:NODE)-[:REL]->(m) RETURN m", nil)
	for _, row := range data {
		fmt.Printf("NODE: %#v\n", row[0].(graph.Node)) // Prints all nodes
	}

	result, _ = conn.ExecNeo(`MATCH (n) DETACH DELETE n`, nil)
	numResult, _ = result.RowsAffected()
	fmt.Printf("Rows Deleted: %d", numResult) // Rows Deleted: 13
}

Slow n' Clean

// Constants to be used throughout the example
const (
	URI          = "bolt://username:password@localhost:7687"
	CreateNode   = "CREATE (n:NODE {foo: {foo}, bar: {bar}})"
	GetNode      = "MATCH (n:NODE) RETURN n.foo, n.bar"
	RelationNode = "MATCH path=(n:NODE)-[:REL]->(m) RETURN path"
	DeleteNodes  = "MATCH (n) DETACH DELETE n"
)

func main() {
	con := createConnection()
	defer con.Close()

	st := prepareSatement(CreateNode, con)
	executeStatement(st)

	st = prepareSatement(GetNode, con)
	rows := queryStatement(st)
	consumeRows(rows, st)

	pipe := preparePipeline(con)
	executePipeline(pipe)

	st = prepareSatement(RelationNode, con)
	rows = queryStatement(st)
	consumeMetadata(rows, st)

	cleanUp(DeleteNodes, con)
}

func createConnection() bolt.Conn {
	driver := bolt.NewDriver()
	con, err := driver.OpenNeo(URI)
	handleError(err)
	return con
}

// Here we prepare a new statement. This gives us the flexibility to
// cancel that statement without any request sent to Neo
func prepareSatement(query string, con bolt.Conn) bolt.Stmt {
	st, err := con.PrepareNeo(query)
	handleError(err)
	return st
}

// Here we prepare a new pipeline statement for running multiple
// queries concurrently
func preparePipeline(con bolt.Conn) bolt.PipelineStmt {
	pipeline, err := con.PreparePipeline(
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(f:FOO)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(b:BAR)",
		"MATCH (n:NODE) CREATE (n)-[:REL]->(z:BAZ)",
	)
	handleError(err)
	return pipeline
}

func executePipeline(pipeline bolt.PipelineStmt) {
	pipelineResults, err := pipeline.ExecPipeline(nil, nil, nil, nil, nil, nil)
	handleError(err)

	for _, result := range pipelineResults {
		numResult, _ := result.RowsAffected()
		fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 2 (per each iteration)
	}

	err = pipeline.Close()
	handleError(err)
}

func queryStatement(st bolt.Stmt) bolt.Rows {
	// Even once I get the rows, if I do not consume them and close the
	// rows, Neo will discard and not send the data
	rows, err := st.QueryNeo(nil)
	handleError(err)
	return rows
}

func consumeMetadata(rows bolt.Rows, st bolt.Stmt) {
	// Here we loop through the rows until we get the metadata object
	// back, meaning the row stream has been fully consumed

	var err error
	err = nil

	for err == nil {
		var row []interface{}
		row, _, err = rows.NextNeo()
		if err != nil && err != io.EOF {
			panic(err)
		} else if err != io.EOF {
			fmt.Printf("PATH: %#v\n", row[0].(graph.Path)) // Prints all paths
		}
	}
	st.Close()
}

func consumeRows(rows bolt.Rows, st bolt.Stmt) {
	// This interface allows you to consume rows one-by-one, as they
	// come off the bolt stream. This is more efficient especially
	// if you're only looking for a particular row/set of rows, as
	// you don't need to load up the entire dataset into memory
	data, _, err := rows.NextNeo()
	handleError(err)

	// This query only returns 1 row, so once it's done, it will return
	// the metadata associated with the query completion, along with
	// io.EOF as the error
	_, _, err = rows.NextNeo()
	handleError(err)
	fmt.Printf("COLUMNS: %#v\n", rows.Metadata()["fields"].([]interface{})) // COLUMNS: n.foo,n.bar
	fmt.Printf("FIELDS: %d %f\n", data[0].(int64), data[1].(float64))       // FIELDS: 1 2.2

	st.Close()
}

// Executing a statement just returns summary information
func executeStatement(st bolt.Stmt) {
	result, err := st.ExecNeo(map[string]interface{}{"foo": 1, "bar": 2.2})
	handleError(err)
	numResult, err := result.RowsAffected()
	handleError(err)
	fmt.Printf("CREATED ROWS: %d\n", numResult) // CREATED ROWS: 1

	// Closing the statment will also close the rows
	st.Close()
}

func cleanUp(query string, con bolt.Conn) {
	result, _ := con.ExecNeo(query, nil)
	fmt.Println(result)
	numResult, _ := result.RowsAffected()
	fmt.Printf("Rows Deleted: %d", numResult) // Rows Deleted: 13
}

// Here we create a simple function that will take care of errors, helping with some code clean up
func handleError(err error) {
	if err != nil {
		panic(err)
	}
}

API

There is much more detailed information in the godoc

This implementation attempts to follow the best practices as per the Bolt specification, but also implements compatibility with Golang's sql.driver interface.

As such, these interfaces closely match the sql.driver interfaces, but they also provide Neo4j Bolt specific functionality in addition to the sql.driver interface.

It is recommended that you use the Neo4j Bolt-specific interfaces if possible. The implementation is more efficient and can more closely support the Neo4j Bolt feature set.

The URL format is: bolt://(user):(password)@(host):(port) Schema must be bolt. User and password is only necessary if you are authenticating.

Connection pooling is provided out of the box with the NewDriverPool method. You can give it the maximum number of connections to have at a time.

You can get logs from the driver by setting the log level using the log packages SetLevel.

Dev Quickstart

# Put in git hooks
ln -s ../../scripts/pre-commit .git/hooks/pre-commit
ln -s ../../scripts/pre-push .git/hooks/pre-push

# No special build steps necessary
go build

# Testing with log info and a local bolt DB, getting coverage output
BOLT_DRIVER_LOG=info NEO4J_BOLT=bolt://localhost:7687 go test -coverprofile=./tmp/cover.out -coverpkg=./... -v -race && go tool cover -html=./tmp/cover.out

# Testing with trace output for debugging
BOLT_DRIVER_LOG=trace NEO4J_BOLT=bolt://localhost:7687 go test -v -race

# Testing with running recorder to record tests for CI
BOLT_DRIVER_LOG=trace NEO4J_BOLT=bolt://localhost:7687 RECORD_OUTPUT=1 go test -v -race

The tests are written in an integration testing style. Most of them are in the statement tests, but should be made more granular in the future.

In order to get CI, I made a recorder mechanism so you don't need to run neo4j alongside the tests in the CI server. You run the tests locally against a neo4j instance with the RECORD_OUTPUT=1 environment variable, it generates the recordings in the ./recordings folder. This is necessary if the tests have changed, or if the internals have significantly changed. Installing the git hooks will run the tests automatically on push. If there are updated tests, you will need to re-run the recorder to add them and push them as well.

You need access to a running Neo4J database to develop for this project, so that you can run the tests to generate the recordings. For the recordings to be generated correctly you also need to make sure authorization is turned off on the Neo4J instance. For more information on Neo4J installation and configuration see the official Neo4j docs: https://neo4j.com/docs/operations-manual/current/installation/

Supported Builds

• Linux (1.4.x, 1.5.x, 1.6.x, 1.7.x, 1.8.x, 1.9.x and tip)
• MacOs (1.7.x, 1.8.x, 1.9.x and tip) according to https://github.com/golang/go/issues/17824, go 1.6.4 (and anything prior does not support osx 10.12) which results in unpredicable behavior (sometimes it is okay, sometimes build hangs, and sometimes build fail due to segfault)

TODO

• Cypher Parser to implement NumInput and pre-flight checking
• More Tests
• Benchmark Tests

Documentation

Overview

Package golangNeo4jBoltDriver implements a driver for the Neo4J Bolt Protocol.

The driver is compatible with Golang's sql.driver interface, but aims to implement a more complete featureset in line with what Neo4J and Bolt provides.

As such, there are multiple interfaces the user can choose from. It's highly recommended that the user use the Neo4J-specific interfaces as they are more flexible and efficient than the provided sql.driver compatible methods.

The interface tries to be consistent throughout. The sql.driver interfaces are standard, but the Neo4J-specific ones contain a naming convention of either "Neo" or "Pipeline".

The "Neo" ones are the basic interfaces for making queries to Neo4j and it's expected that these would be used the most.

The "Pipeline" ones are to support Bolt's pipelining features. Pipelines allow the user to send Neo4j many queries at once and have them executed by the database concurrently. This is useful if you have a bunch of queries that aren't necessarily dependant on one another, and you want to get better performance. The internal APIs will also pipeline statements where it is able to reliably do so, but by manually using the pipelining feature you can maximize your throughput.

The API provides connection pooling using the `NewDriverPool` method. This allows you to pass it the maximum number of open connections to be used in the pool. Once this limit is hit, any new clients will have to wait for a connection to become available again.

The sql driver is registered as "neo4j-bolt". The sql.driver interface is much more limited than what bolt and neo4j supports. In some cases, concessions were made in order to make that interface work with the neo4j way of doing things. The main instance of this is the marshalling of objects to/from the sql.driver.Value interface. In order to support object types that aren't supported by this interface, the internal encoding package is used to marshal these objects to byte strings. This ultimately makes for a less efficient and more 'clunky' implementation. A glaring instance of this is passing parameters. Neo4j expects named parameters but the driver interface can only really support positional parameters. To get around this, the user must create a map[string]interface{} of their parameters and marshal it to a driver.Value using the encoding.Marshal function. Similarly, the user must unmarshal data returned from the queries using the encoding.Unmarshal function, then use type assertions to retrieve the proper type.

In most cases the driver will return the data from neo as the proper go-specific types. For integers they always come back as int64 and floats always come back as float64. This is for the convenience of the user and acts similarly to go's JSON interface. This prevents the user from having to use reflection to get these values. Internally, the types are always transmitted over the wire with as few bytes as possible.

There are also cases where no go-specific type matches the returned values, such as when you query for a node, relationship, or path. The driver exposes specific structs which represent this data in the 'structures.graph' package. There are 4 types - Node, Relationship, UnboundRelationship, and Path. The driver returns interface{} objects which must have their types properly asserted to get the data out.

There are some limitations to the types of collections the driver supports. Specifically, maps should always be of type map[string]interface{} and lists should always be of type []interface{}. It doesn't seem that the Bolt protocol supports uint64 either, so the biggest number it can send right now is the int64 max.

The URL format is: `bolt://(user):(password)@(host):(port)` Schema must be `bolt`. User and password is only necessary if you are authenticating. TLS is supported by using query parameters on the connection string, like so: `bolt://host:port?tls=true&tls_no_verify=false`

The supported query params are:

* timeout - the number of seconds to set the connection timeout to. Defaults to 60 seconds. * tls - Set to 'true' or '1' if you want to use TLS encryption * tls_no_verify - Set to 'true' or '1' if you want to accept any server certificate (for testing, not secure) * tls_ca_cert_file - path to a custom ca cert for a self-signed TLS cert * tls_cert_file - path to a cert file for this client (need to verify this is processed by Neo4j) * tls_key_file - path to a key file for this client (need to verify this is processed by Neo4j)

Errors returned from the API support wrapping, so if you receive an error from the library, it might be wrapping other errors. You can get the innermost error by using the `InnerMost` method. Failure messages from Neo4J are reported, along with their metadata, as an error. In order to get the failure message metadata from a wrapped error, you can do so by calling `err.(*errors.Error).InnerMost().(messages.FailureMessage).Metadata`

If there is an error with the database connection, you should get a sql/driver ErrBadConn as per the best practice recommendations of the Golang SQL Driver. However, this error may be wrapped, so you might have to call `InnerMost` to get it, as specified above.

Index

• Variables
• type ClosableDriverPool
  • func NewClosableDriverPool(connStr string, max int) (ClosableDriverPool, error)
• type Conn
• type Driver
  • func NewDriver() Driver
• type DriverPool
  • func NewDriverPool(connStr string, max int) (DriverPool, error)
• type Event
• type PipelineRows
• type PipelineStmt
• type Result
• type Rows
• type Stmt
• type Tx

Variables

var (

	// Version is the current version of this driver
	Version = "1.0"
	// ClientID is the id of this client
	ClientID = "GolangNeo4jBolt/" + Version
)

Types

type ClosableDriverPool

type ClosableDriverPool interface {
	DriverPool
	Close() error
}

ClosableDriverPool like the DriverPool but with a closable function

func NewClosableDriverPool

func NewClosableDriverPool(connStr string, max int) (ClosableDriverPool, error)

NewClosableDriverPool create a closable driver pool

type Conn

type Conn interface {
	// PrepareNeo prepares a neo4j specific statement
	PrepareNeo(query string) (Stmt, error)
	// PreparePipeline prepares a neo4j specific pipeline statement
	// Useful for running multiple queries at the same time
	PreparePipeline(query ...string) (PipelineStmt, error)
	// QueryNeo queries using the neo4j-specific interface
	QueryNeo(query string, params map[string]interface{}) (Rows, error)
	// QueryNeoAll queries using the neo4j-specific interface and returns all row data and output metadata
	QueryNeoAll(query string, params map[string]interface{}) ([][]interface{}, map[string]interface{}, map[string]interface{}, error)
	// QueryPipeline queries using the neo4j-specific interface
	// pipelining multiple statements
	QueryPipeline(query []string, params ...map[string]interface{}) (PipelineRows, error)
	// ExecNeo executes a query using the neo4j-specific interface
	ExecNeo(query string, params map[string]interface{}) (Result, error)
	// ExecPipeline executes a query using the neo4j-specific interface
	// pipelining multiple statements
	ExecPipeline(query []string, params ...map[string]interface{}) ([]Result, error)
	// Close closes the connection
	Close() error
	// Begin starts a new transaction
	Begin() (driver.Tx, error)
	// SetChunkSize is used to set the max chunk size of the
	// bytes to send to Neo4j at once
	SetChunkSize(uint16)
	// SetTimeout sets the read/write timeouts for the
	// connection to Neo4j
	SetTimeout(time.Duration)
}

Conn represents a connection to Neo4J

Implements a neo-friendly interface. Some of the features of this interface implement neo-specific features unavailable in the sql/driver compatible interface

Conn objects, and any prepared statements/transactions within ARE NOT THREAD SAFE. If you want to use multipe go routines with these objects, you should use a driver to create a new conn for each routine.

type Driver

type Driver interface {
	// Open opens a sql.driver compatible connection. Used internally
	// by the go sql interface
	Open(string) (driver.Conn, error)
	// OpenNeo opens a Neo-specific connection. This should be used
	// directly when not using the golang sql interface
	OpenNeo(string) (Conn, error)
}

Driver is a driver allowing connection to Neo4j The driver allows you to open a new connection to Neo4j

Implements sql/driver, but also includes its own more neo-friendly interface. Some of the features of this interface implement neo-specific features unavailable in the sql/driver compatible interface

Driver objects should be THREAD SAFE, so you can use them to open connections in multiple threads. The connection objects themselves, and any prepared statements/transactions within ARE NOT THREAD SAFE.

func NewDriver

func NewDriver() Driver

NewDriver creates a new Driver object

type DriverPool

type DriverPool interface {
	// OpenPool opens a Neo-specific connection.
	OpenPool() (Conn, error)
	// contains filtered or unexported methods
}

DriverPool is a driver allowing connection to Neo4j with support for connection pooling The driver allows you to open a new connection to Neo4j

Driver objects should be THREAD SAFE, so you can use them to open connections in multiple threads. The connection objects themselves, and any prepared statements/transactions within ARE NOT THREAD SAFE.

func NewDriverPool

func NewDriverPool(connStr string, max int) (DriverPool, error)

NewDriverPool creates a new Driver object with connection pooling

type Event

type Event struct {
	Timestamp int64 `json:"-"`
	Event     []byte
	IsWrite   bool
	Completed bool
	Error     error
}

Event represents a single recording (read or write) event in the recorder

type PipelineRows

type PipelineRows interface {
	// Columns Gets the names of the columns in the returned dataset
	Columns() []string
	// Metadata Gets all of the metadata returned from Neo on query start
	Metadata() map[string]interface{}
	// Close the rows, flushing any existing datastream
	Close() error
	// NextPipeline gets the next row result
	// When the rows are completed, returns the success metadata and the next
	// set of rows.
	// When all rows are completed, returns io.EOF
	NextPipeline() ([]interface{}, map[string]interface{}, PipelineRows, error)
}

PipelineRows represents results of a set of rows from the DB when running a pipeline statement.

Row objects ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type PipelineStmt

type PipelineStmt interface {
	// Close Closes the statement. See sql/driver.Stmt.
	Close() error
	// ExecPipeline executes a set of queries that returns no rows.
	ExecPipeline(params ...map[string]interface{}) ([]Result, error)
	// QueryPipeline executes a set of queries that return data.
	// Implements a Neo-friendly alternative to sql/driver.
	QueryPipeline(params ...map[string]interface{}) (PipelineRows, error)
}

PipelineStmt represents a set of statements to run against the database

PipelineStmt objects, and any rows prepared within ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type Result

type Result interface {
	// LastInsertId Always returns -1. This is necessary
	// to meet the sql.driver interface
	LastInsertId() (int64, error)
	// RowsAffected returns the number of rows affected
	// This doesn't currently support updates, only
	// inserts/deletions
	RowsAffected() (int64, error)
	// Metadata returns the metadata response from neo4j
	Metadata() map[string]interface{}
}

Result represents a result from a query that returns no data

type Rows

type Rows interface {
	// Columns Gets the names of the columns in the returned dataset
	Columns() []string
	// Metadata Gets all of the metadata returned from Neo on query start
	Metadata() map[string]interface{}
	// Close the rows, flushing any existing datastream
	Close() error
	// NextNeo gets the next row result
	// When the rows are completed, returns the success metadata
	// and io.EOF
	NextNeo() ([]interface{}, map[string]interface{}, error)
	// All gets all of the results from the row set. It's recommended to use NextNeo when
	// there are a lot of rows
	All() ([][]interface{}, map[string]interface{}, error)
}

Rows represents results of rows from the DB

Row objects ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type Stmt

type Stmt interface {
	// Close Closes the statement. See sql/driver.Stmt.
	Close() error
	// ExecNeo executes a query that returns no rows. Implements a Neo-friendly alternative to sql/driver.
	ExecNeo(params map[string]interface{}) (Result, error)
	// QueryNeo executes a query that returns data. Implements a Neo-friendly alternative to sql/driver.
	QueryNeo(params map[string]interface{}) (Rows, error)
}

Stmt represents a statement to run against the database

Stmt objects, and any rows prepared within ARE NOT THREAD SAFE. If you want to use multiple go routines with these objects, you should use a driver to create a new conn for each routine.

type Tx

type Tx interface {
	// Commit commits the transaction
	Commit() error
	// Rollback rolls back the transaction
	Rollback() error
}

Tx represents a transaction