datastore

README

Datastore Wrapper

🚧 This package is unstable 🚧 github repo

(AppEngine | Cloud) Datastore wrapper for Go 👉

Simple. Happy. Respect standard library.

$ go get -u go.mercari.io/datastore

Feature

DO

• Wrap google.golang.org/appengine/datastore and cloud.google.com/go/datastore
  • keep key behavior
  • align to cloud.google.com/go/datastore first
• Re-implement datastore package
• Re-implement datastore.SaveStruct & LoadStruct
  • Ignore unmapped property
  • Add PropertyTranslator interface
    • Convert types like mytime.Unix to time.Time and reverse it
    • Rename property like CreatedAt to createdAt or created_at and reverse it
• Re-implement PropertyLoadSaver
  • Pass context.Context to Save & Load method
  • It can catch entity from the cache layers as well as from the datastore
• Add retry feature to each RPC
  • e.g. Retry AllocateID when it failed
• Add cache layer uses without Tx
  • About...
    • Local Cache
    • Memcache
    • etc...
  • Easy to ON/OFF switching
• Add RPC hook layer
  • It makes implements memvache like cache strategy
• Add some useful methods
  • GetAsIntIDMap, GetAsStringIDMap
  • aedatastore/TransactionContext

DON'T

• have utility functions
• support firestore

Restriction

• aedatastore package
  • When using slice of struct, MUST specified datastore:",flatten" option.
    • original (ae & cloud) datastore.SaveStruct have different behaviors.
    • see aeprodtest/main.go /api/test3

TODO

• Write tests for namespace
• Implement cache layers
• Retry feature

Committers

• Masahiro Wakame (@vvakame)

Contribution

Please read the CLA below carefully before submitting your contribution.

https://www.mercari.com/cla/

Setup environment & Run tests

• requirements
  • gcloud sdk
    • gcloud components install app-engine-go
    • gcloud components install beta cloud-datastore-emulator
  • dep
    • go get -u github.com/golang/dep/cmd/dep

$ ./setup.sh # exec once
$ ./serve.sh # exec in background
$ ./test.sh

$ ./build-in-docker.sh

License

Copyright 2017 Mercari, Inc.

Licensed under the MIT License.

Documentation

Index

• Variables
• func LoadStruct(ctx context.Context, dst interface{}, p []Property) error
• type Batch
  • func (b *Batch) Delete(key Key) chan error
  • func (b *Batch) Exec(ctx context.Context) error
  • func (b *Batch) Get(key Key, dst interface{}) chan error
  • func (b *Batch) Put(key Key, src interface{}) chan *PutResult
  • func (b *Batch) UnwrapPutResult(r *PutResult) (Key, error)
• type Client
• type ClientGenerator
• type ClientOption
  • func WithCredentialsFile(filename string) ClientOption
  • func WithHTTPClient(client *http.Client) ClientOption
  • func WithProjectID(projectID string) ClientOption
  • func WithScopes(scope ...string) ClientOption
  • func WithTokenSource(s oauth2.TokenSource) ClientOption
• type Commit
• type Cursor
• type Entity
• type ErrFieldMismatch
  • func (e *ErrFieldMismatch) Error() string
• type GeoPoint
• type Iterator
• type Key
• type KeyLoader
• type MultiError
  • func (m MultiError) Error() string
• type PendingKey
• type Property
  • func SaveStruct(ctx context.Context, src interface{}) ([]Property, error)
• type PropertyList
  • func (l *PropertyList) Load(ctx context.Context, p []Property) error
  • func (l *PropertyList) Save(ctx context.Context) ([]Property, error)
• type PropertyLoadSaver
• type PropertyTranslator
• type PutResult
• type Query
• type Transaction
• type TransactionBatch
  • func (b *TransactionBatch) Delete(key Key) chan error
  • func (b *TransactionBatch) Exec()
  • func (b *TransactionBatch) Get(key Key, dst interface{}) chan error
  • func (b *TransactionBatch) Put(key Key, src interface{}) chan *TransactionPutResult
  • func (b *TransactionBatch) UnwrapPutResult(r *TransactionPutResult) (PendingKey, error)
• type TransactionPutResult

Variables

var (
	ErrInvalidEntityType = errors.New("datastore: invalid entity type")
	ErrInvalidKey        = errors.New("datastore: invalid key")
	ErrNoSuchEntity      = errors.New("datastore: no such entity")
)

var ErrConcurrentTransaction = errors.New("datastore: concurrent transaction")

var LoadEntity = loadEntity

TODO hide LoadEntity project outside

var SaveEntity = saveEntity

var SuppressErrFieldMismatch = true

Functions

func LoadStruct

func LoadStruct(ctx context.Context, dst interface{}, p []Property) error

LoadStruct loads the properties from p to dst. dst must be a struct pointer.

The values of dst's unmatched struct fields are not modified, and matching slice-typed fields are not reset before appending to them. In particular, it is recommended to pass a pointer to a zero valued struct on each LoadStruct call.

Types

type Batch

type Batch struct {
	Client Client
	// contains filtered or unexported fields
}

func (*Batch) Delete

func (b *Batch) Delete(key Key) chan error

func (*Batch) Exec

func (b *Batch) Exec(ctx context.Context) error

func (*Batch) Get

func (b *Batch) Get(key Key, dst interface{}) chan error

func (*Batch) Put

func (b *Batch) Put(key Key, src interface{}) chan *PutResult

func (*Batch) UnwrapPutResult

func (b *Batch) UnwrapPutResult(r *PutResult) (Key, error)

type Client

type Client interface {
	NewTransaction(ctx context.Context) (Transaction, error)
	RunInTransaction(ctx context.Context, f func(tx Transaction) error) (Commit, error)
	Run(ctx context.Context, q Query) Iterator
	AllocatedIDs(ctx context.Context, keys []Key) ([]Key, error)
	Count(ctx context.Context, q Query) (int, error)
	GetAll(ctx context.Context, q Query, dst interface{}) ([]Key, error)

	IncompleteKey(kind string, parent Key) Key
	NameKey(kind, name string, parent Key) Key
	IDKey(kind string, id int64, parent Key) Key

	NewQuery(kind string) Query

	Close() error

	DecodeCursor(s string) (Cursor, error)

	Batch() *Batch
	// contains filtered or unexported methods
}

type ClientGenerator

type ClientGenerator func(ctx context.Context, opts ...ClientOption) (Client, error)

var FromContext ClientGenerator

type ClientOption

type ClientOption interface {
	Apply(*internal.ClientSettings)
}

func WithCredentialsFile

func WithCredentialsFile(filename string) ClientOption

WithCredentialsFile returns a ClientOption that authenticates API calls with the given service account or refresh token JSON credentials file.

func WithHTTPClient

func WithHTTPClient(client *http.Client) ClientOption

WithHTTPClient returns a ClientOption that specifies the HTTP client to use as the basis of communications. This option may only be used with services that support HTTP as their communication transport. When used, the WithHTTPClient option takes precedent over all other supplied options.

func WithProjectID

func WithProjectID(projectID string) ClientOption

func WithScopes

func WithScopes(scope ...string) ClientOption

WithScopes returns a ClientOption that overrides the default OAuth2 scopes to be used for a service.

func WithTokenSource

func WithTokenSource(s oauth2.TokenSource) ClientOption

WithTokenSource returns a ClientOption that specifies an OAuth2 token source to be used as the basis for authentication.

type Commit

type Commit interface {
	Key(p PendingKey) Key
}

type Cursor

type Cursor interface {
	String() string
}

type Entity

type Entity struct {
	Key        Key
	Properties []Property
}

type ErrFieldMismatch

type ErrFieldMismatch struct {
	StructType reflect.Type
	FieldName  string
	Reason     string
}

ErrFieldMismatch is returned when a field is to be loaded into a different type than the one it was stored from, or when a field is missing or unexported in the destination struct. StructType is the type of the struct pointed to by the destination argument passed to Get or to Iterator.Next.

func (*ErrFieldMismatch) Error

func (e *ErrFieldMismatch) Error() string

type GeoPoint

type GeoPoint struct {
	Lat, Lng float64
}

type Iterator

type Iterator interface {
	Next(dst interface{}) (Key, error)
	Cursor() (Cursor, error)
}

type Key

type Key interface {
	Kind() string
	ID() int64
	Name() string
	ParentKey() Key
	Namespace() string

	String() string
	GobEncode() ([]byte, error)
	GobDecode(buf []byte) error
	MarshalJSON() ([]byte, error)
	UnmarshalJSON(buf []byte) error
	Encode() string
}

type KeyLoader

type KeyLoader interface {
	PropertyLoadSaver
	LoadKey(ctx context.Context, k Key) error
}

type MultiError

type MultiError []error

MultiError is returned by batch operations when there are errors with particular elements. Errors will be in a one-to-one correspondence with the input elements; successful elements will have a nil entry.

func (MultiError) Error

func (m MultiError) Error() string

type PendingKey

type PendingKey interface {
	StoredContext() context.Context
}

type Property

type Property struct {
	Name    string
	Value   interface{}
	NoIndex bool
}

func SaveStruct

func SaveStruct(ctx context.Context, src interface{}) ([]Property, error)

SaveStruct returns the properties from src as a slice of Properties. src must be a struct pointer.

type PropertyList

type PropertyList []Property

func (*PropertyList) Load

func (l *PropertyList) Load(ctx context.Context, p []Property) error

Load loads all of the provided properties into l. It does not first reset *l to an empty slice.

func (*PropertyList) Save

func (l *PropertyList) Save(ctx context.Context) ([]Property, error)

Save saves all of l's properties as a slice of Properties.

type PropertyLoadSaver

type PropertyLoadSaver interface {
	Load(ctx context.Context, ps []Property) error
	Save(ctx context.Context) ([]Property, error)
}

type PropertyTranslator

type PropertyTranslator interface {
	ToPropertyValue(ctx context.Context) (interface{}, error)
	FromPropertyValue(ctx context.Context, p Property) (dst interface{}, err error)
}

type PutResult

type PutResult struct {
	Key Key
	Err error
}

type Query

type Query interface {
	Ancestor(ancestor Key) Query
	EventualConsistency() Query
	Namespace(ns string) Query
	Transaction(t Transaction) Query
	Filter(filterStr string, value interface{}) Query
	Order(fieldName string) Query
	Project(fieldNames ...string) Query
	Distinct() Query
	// NOT IMPLEMENTED ON APPENGINE DistinctOn(fieldNames ...string) *Query
	KeysOnly() Query
	Limit(limit int) Query
	Offset(offset int) Query
	Start(c Cursor) Query
	End(c Cursor) Query
}

type Transaction

type Transaction interface {
	Get(key Key, dts interface{}) error
	GetMulti(keys []Key, dst interface{}) error
	Put(key Key, src interface{}) (PendingKey, error)
	PutMulti(keys []Key, src interface{}) ([]PendingKey, error)
	Delete(key Key) error
	DeleteMulti(keys []Key) error

	Commit() (Commit, error)
	Rollback() error

	Batch() *TransactionBatch
}

type TransactionBatch

type TransactionBatch struct {
	Transaction Transaction
	// contains filtered or unexported fields
}

func (*TransactionBatch) Delete

func (b *TransactionBatch) Delete(key Key) chan error

func (*TransactionBatch) Exec

func (b *TransactionBatch) Exec()

func (*TransactionBatch) Get

func (b *TransactionBatch) Get(key Key, dst interface{}) chan error

func (*TransactionBatch) Put

func (b *TransactionBatch) Put(key Key, src interface{}) chan *TransactionPutResult

func (*TransactionBatch) UnwrapPutResult

func (b *TransactionBatch) UnwrapPutResult(r *TransactionPutResult) (PendingKey, error)

type TransactionPutResult

type TransactionPutResult struct {
	PendingKey PendingKey
	Err        error
}