dsquery

README

dsquery

This is a simple "query builder / executor" for Google's Datastore.

For the moment it is simple, but I would like to build it out with some better features.

The primary purpose right now is to make some go based queries more readable or at least more consistent especially.

How it works

All this library does is provide wrappers which can be used to wrap existing datastore queries.

It allows you to preform operations like "and" or "or" between two queries in a more standard way.

It runs key-only queries meaning you will need to query the keys with the resulting query yourself with a GetMulti.

Examples

For a runnable demonstration, see the command in cmd/dsquery-example.

Given this data structure:

package example

import (
	"cloud.google.com/go/datastore"
)

const (
	FruitKind = "Fruit"
)

type Fruit struct {
	DSKey         *datastore.Key `datastore:"__key__"`
	Name          string         `datastore:",noindex"`
	Color         string
	Producers     []string
}

And the data

Key	Name	Color	Producers
123	Potato	Brown	[Ireland, Peru, Scotland]
234	Orange	Orange	[USA, Brazil, China]
345	Chilli	Red	[Mexico, Turkey, China]

Say for the query:

• All Fruits that are Orange or Red

    fruitQuery := &dsquery.Or{
        Name: "root fruit query",
		Queries:    []*datastore.Query{
            datastore.NewQuery(FruitKind).FilterField("Color", "=", "Orange"),
            datastore.NewQuery(FruitKind).FilterField("Color", "=", "Red"),
        },
		SubQueries: nil,
    }

	keys, err := fruitQuery.Query(dsClient, request.Context())
    if err != nil {
        logger.Errorf("Query error %v", err)
        return
    }

	logger.Infof("Running GetMulti for %d fruit", len(keys))
	fruits := make([]*Fruit, len(keys), len(keys))
	if err := dsClient.GetMulti(request.Context(), keys, fruits); err != nil {
		logger.Errorf("GetMulti error %v", err)
		return err
	}

Should return: Orange and Chilli

Say for the query:

• All Fruits that are (Orange or Red) AND (Grown in China OR USA)

    fruitQuery := &dsquery.And{
        Name: "root fruit query",
        Queries:    []*datastore.Query{},
        SubQueries: []dsquery.Query{
            &dsquery.Or{
                Name: "color query",
                Queries:    []*datastore.Query{
                datastore.NewQuery(FruitKind).FilterField("Color", "=", "Orange"),
                datastore.NewQuery(FruitKind).FilterField("Color", "=", "Red"),
            },
            &dsquery.Or{
                Name: "country query",
                Queries:    []*datastore.Query{
                datastore.NewQuery(FruitKind).FilterField("Producers", "=", "USA"),
                datastore.NewQuery(FruitKind).FilterField("Producers", "=", "China"),
            },
        },
    }

	keys, err := fruitQuery.Query(dsClient, request.Context())
    if err != nil {
        logger.Errorf("Query error %v", err)
        return
    }

	logger.Infof("Running GetMulti for %d fruit", len(keys))
	fruits := make([]*Fruit, len(keys), len(keys))
	if err := dsClient.GetMulti(request.Context(), keys, fruits); err != nil {
		logger.Errorf("GetMulti error %v", err)
		return err
	}

Should return: Orange and Chilli

Notes

Happy to receive PRs.. Especially around documentation and testing.

License

This project is licensed under the MIT License.

Documentation

Overview

Package dsquery provides helpers for composing and executing Google Cloud Datastore queries. It wraps datastore.Query objects with logical operators such as AND/OR and supports optional caching of results.

Index

• func DSKeyMapMergeAnd(m map[string]*datastore.Key, keys []*datastore.Key) map[string]*datastore.Key
• func DSKeyMapMergeNot(m map[string]*datastore.Key, keys []*datastore.Key) map[string]*datastore.Key
• func ExtractMapStringKeysKey(m map[string]*datastore.Key) []*datastore.Key
• type And
  • func (qa *And) Len() int
  • func (qa *And) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)
• type Base
• type Cached
  • func (c *Cached) Len() int
  • func (c *Cached) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)
• type DatastoreClient
• type Ident
  • func (qi *Ident) Len() int
  • func (qi *Ident) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)
• type Not
  • func (qn *Not) Len() int
  • func (qn *Not) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)
• type Or
  • func (qo *Or) Len() int
  • func (qo *Or) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)
• type Query

Examples

• And
• Cached
• Or

Functions

func DSKeyMapMergeAnd

func DSKeyMapMergeAnd(m map[string]*datastore.Key, keys []*datastore.Key) map[string]*datastore.Key

Helper function to merge a set with an array, and produce a set of datastore keys.

func DSKeyMapMergeNot

func DSKeyMapMergeNot(m map[string]*datastore.Key, keys []*datastore.Key) map[string]*datastore.Key

Helper function to remove keys from a set and produce a new set of datastore keys.

func ExtractMapStringKeysKey

func ExtractMapStringKeysKey(m map[string]*datastore.Key) []*datastore.Key

Extracts all the datastore keys from a `map[string]*datastore.Key`

Types

type And

type And Base

AND Query

Example

ExampleAnd demonstrates using And with subqueries.

ds := &staticDS{results: [][]*datastore.Key{
	{datastore.NameKey("Fruit", "1", nil), datastore.NameKey("Fruit", "2", nil), datastore.NameKey("Fruit", "3", nil)},
	{datastore.NameKey("Fruit", "2", nil), datastore.NameKey("Fruit", "3", nil), datastore.NameKey("Fruit", "4", nil)},
}}

q := &And{Queries: []*datastore.Query{
	datastore.NewQuery("Fruit").FilterField("Color", "=", "Red"),
	datastore.NewQuery("Fruit").FilterField("Producers", "=", "USA"),
}}

keys, _ := q.Query(ds, context.Background())
sort.Slice(keys, func(i, j int) bool { return keys[i].Name < keys[j].Name })
for _, k := range keys {
	fmt.Println(k.Name)
}

Output:

2
3

func (*And) Len

func (qa *And) Len() int

Count of all queries

func (*And) Query

func (qa *And) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)

Query function

type Base

type Base struct {
	// Data store queries to run
	Queries []*datastore.Query
	// Sub queries after that to run
	SubQueries []Query
	// Provided for your convenience when debugging
	Name string
}

Base entity

type Cached

type Cached struct {
	StoredQuery   Query
	StoredResults []*datastore.Key
	Name          string
	TTL           time.Duration
	Expiration    time.Time
	sync.RWMutex
}

An object that contains just a single query, provides thread safe caching

Example

ExampleCached shows how Cached prevents a query from running more than once.

q := &countQuery{keys: []*datastore.Key{datastore.NameKey("Fruit", "1", nil)}}
cached := &Cached{StoredQuery: q}

_, _ = cached.Query(nil, context.Background())
_, _ = cached.Query(nil, context.Background())

fmt.Println(q.n)

Output:

1

func (*Cached) Len

func (c *Cached) Len() int

Count of all queries

func (*Cached) Query

func (c *Cached) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)

Query function

type DatastoreClient

type DatastoreClient interface {
	// GetAll should perform `q` against the underlying datastore and return
	// every matching key. Implementations may ignore `dst` as queries in
	// this package are executed in keys-only mode.
	GetAll(ctx context.Context, q *datastore.Query, dst interface{}) (keys []*datastore.Key, err error)
}

DatastoreClient exposes the datastore operations required by this package. The real *datastore.Client type already satisfies this interface. Custom or test implementations should execute the provided query and return all resulting keys. The `dst` parameter is kept for signature compatibility with datastore.Client.GetAll and may be nil.

type Ident

type Ident struct {
	StoredQuery *datastore.Query
	Name        string
}

An object that contains just a single query, provided for debugging or intentinally ordering queries in a particular way

func (*Ident) Len

func (qi *Ident) Len() int

Count of all queries

func (*Ident) Query

func (qi *Ident) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)

Query function

type Not

type Not Base

The NOT query

func (*Not) Len

func (qn *Not) Len() int

Count of all queries

func (*Not) Query

func (qn *Not) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)

Query function

type Or

type Or Base

The OR query

Example

ExampleOr demonstrates combining multiple queries with Or.

ds := &staticDS{results: [][]*datastore.Key{
	{datastore.NameKey("Fruit", "1", nil), datastore.NameKey("Fruit", "2", nil)},
	{datastore.NameKey("Fruit", "3", nil)},
}}

q := &Or{Queries: []*datastore.Query{
	datastore.NewQuery("Fruit").FilterField("Color", "=", "Brown"),
	datastore.NewQuery("Fruit").FilterField("Color", "=", "Orange"),
}}

keys, _ := q.Query(ds, context.Background())
sort.Slice(keys, func(i, j int) bool { return keys[i].Name < keys[j].Name })
for _, k := range keys {
	fmt.Println(k.Name)
}

Output:

1
2
3

func (*Or) Len

func (qo *Or) Len() int

Count of all queries

func (*Or) Query

func (qo *Or) Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)

Query function

type Query

type Query interface {
	// Query function runs the queries as per data structure
	Query(dsClient DatastoreClient, ctx context.Context) ([]*datastore.Key, error)
	// Count of all queries
	Len() int
}

Query interface