data

README

Data

The data package is designed to provide a consistent programming model for data access regardless of the underlying data store. It contains sub packages that are specific to given database. CockroachDB is the database that is supported by default. Application can enable other databases as long as they are supported by Gorm.

Example Usage

To use this package, include the following code snippet in your application. With these two lines, go-lanai will instantiate all the components provided in the data package, as well as the components specifically for CockroachDB

	data.Use()
	cockroach.Use()

Add the following section in application.yml. These are the connection parameters to your database.

data:
  logging:
    level: warn
    slow-threshold: 5s
  cockroach:
    host: localhost
    port: 26257
    sslmode: disable
    username: my_user_name
    Password: my_password
    database: my_db_name

Define your database model. The following example is a model for a database table called friend that has three columns id, first_name and last_name

type Friend struct {
    ID        uuid.UUID `gorm:"column:id;primary_key;type:UUID;default:gen_random_uuid();"`
    FirstName string    `gorm:"column:first_name;type:text;not null;"`
    LastName  string    `gorm:"column:last_name;type:text;not null;"`
}

Declare a repository for this model. The repo.GormApi interface allows you to write low level database queries. The repo.CrudRepository has convenient methods for CRUD operations.

type FriendsRepository struct {
	repo.GormApi
	repo.CrudRepository
}

func NewFriendRepository(factory repo.Factory) *FriendsRepository {
	crud := factory.NewCRUD(&model.Friend{})

	ret := FriendsRepository{
		CrudRepository: crud,
	}
	if gf, ok := factory.(*repo.GormFactory); ok {
		ret.GormApi = gf.NewGormApi()
	}
	return &ret
}

Make this repository available for dependency injection, and you will be able to use it in your code. (Call this Use() function in your setup code so that it's available for injection).

func Use() {
	bootstrap.AddOptions(
		fx.Provide(
			NewFriendRepository,
		),
	)
}

CRUD Repository

The CrudRepository interface is an abstraction that defines the most commonly used data access operations such as Create, Read, Update, Delete. go-lanai provides implementation for these implementations, so they don't have to be repeated in application code. This is all that is required to instantiate a CrudRepository for a model.

type FriendRepository CrudRepository

func NewFriendRepository(factory Factory) FriendRepository {
    return factory.NewCRUD(&model.Friend{})
}

Most CrudRepository method takes Condition and Options. Conditions are conditional statements that are appended to the query, Options defines how the query should be processed. For example, a query to return all the friends whose first name is John by page can be written with a condition and option.

	var friends []model.Friend

	err = r.FindAllBy(
		ctx,
		&friends,
        &model.Friend{FirstName: "John"},
		repo.Page(pageNumber, pageSize),
	)

Gorm

Sometimes application have data access logic that are beyond the CRUD operations. For these situations, developer can work directly with the lower level gorm API.

	api := factory.NewGormApi(options...)

Error Translation

Error originating from the database driver are mapped to hierarchical DataError. Application code can compare the error they received to the errors defined in the error hierarchy to inspect the error case.

go-lanai also uses this error hierarchy to translate the data access error to web status code, so that if application code returned the error directly as web response the http response status will be correct.

This is how the error handler translate the status code using the error hierarchy. Application code can also use similar technique to inspect the error case.

func (t WebDataErrorTranslator) Translate(ctx context.Context, err error) error {
	//nolint:errorlint
	if _, ok := err.(errorutils.ErrorCoder); !ok || !errors.Is(err, ErrorCategoryData) {
		return err
	}

	switch {
	case errors.Is(err, ErrorRecordNotFound), errors.Is(err, ErrorIncorrectRecordCount):
		return t.errorWithStatusCode(ctx, err, http.StatusNotFound)
	case errors.Is(err, ErrorSubTypeDataIntegrity):
		return t.dataIntegrityErrorWithStatusCode(ctx, err, http.StatusConflict)
	case errors.Is(err, ErrorSubTypeQuery):
		return t.errorWithStatusCode(ctx, err, http.StatusBadRequest)
	case errors.Is(err, ErrorSubTypeTimeout):
		return t.errorWithStatusCode(ctx, err, http.StatusRequestTimeout)
	case errors.Is(err, ErrorTypeTransient):
		return t.errorWithStatusCode(ctx, err, http.StatusServiceUnavailable)
	default:
		return t.errorWithStatusCode(ctx, err, http.StatusInternalServerError)
	}
}

Transaction

The tx package provides two ways for application code that requires transaction. The func Transaction(ctx context.Context, tx TxFunc, opts ...*sql.TxOptions) error function allows application code to provide a function that will be run within a transaction. If this function returns error, any database operation issued within this function will be rolled back. Otherwise, results will be committed.

In this example, if the second operation failed, the first operation will be rolled back.

e = tx.Transaction(ctx, func(ctx context.Context) (err error) {
    // first operation
    firstFriend := model.Friend{firstName:"John", lastName:"Smith"}
    err = di.Repo.Create(ctx, firstFriend)
	if err != nil {
	    return err	
    }
    // second operation
    another := model.Friend{firstName:"Jane", lastName:"Doe"}
    err = di.Repo.Create(ctx, another)
    return err
})

Alternatively, application code can also handle transaction manually using the following set of methods.

// Begin start a transaction. the returned context.Context should be used for any transactional operations
// if returns an error, the returned context.Context should be discarded.
func Begin(ctx context.Context, opts ...*sql.TxOptions) (context.Context, error) 

// Rollback rollbacks a transaction. The returned context.Context is the original provided context when Begin is called.
// if returns an error, the returned context.Context should be discarded
func Rollback(ctx context.Context) (context.Context, error)

// Commit commits a transaction. the returned context.Context is the original provided context when Begin is called.
// if returns an error, the returned context.Context should be discarded
func Commit(ctx context.Context) (context.Context, error)

// SavePoint works with RollbackTo and have to be within a transaction.
// the returned context.Context should be used for any transactional operations between corresponding SavePoint and RollbackTo
// if returns an error, the returned context.Context should be discarded
func SavePoint(ctx context.Context, name string) (context.Context, error)

// RollbackTo works with SavePoint and have to be within a transaction.
// the returned context.Context should be used for any transactional operations between corresponding SavePoint and RollbackTo
// if returns an error, the returned context.Context should be discarded
func RollbackTo(ctx context.Context, name string) (context.Context, error)

Special Data Types

EncryptedMap

EncryptedMap is useful when certain aspect of the data needs to be encrypted. The encryption is backed by Vault transit secret engine.

The following snippet declares a model that has encrypted data.

type EncryptedModel struct {
	ID    int    `gorm:"primaryKey;type:serial;"`
	Name  string `gorm:"uniqueIndex;not null;"`
	Value *EncryptedMap
}

Saving to the database is the same as any other model.

v := map[string]interface{}{
    "key1": "value1",
    "key2": 2.0,
}

kid := uuid.New()

pqcrypt.CreateKeyWithUUID(ctx, kid)

m := EncryptedModel{
    ID: 12345678,
    Name:  "my_encrypted_model",
    Value: NewEncryptedMap(kid, v),
}

myRepo.Save(ctx, &m)

Reading from the database will decrypt the data.

m := EncryptedModel{}
myRepo.FindById(ctx, &m, 12345678) // m's Value field will have the decrypted map

Tenancy

If a model embeds the Tenancy type. This model gets two fields that facilitates multi tenant implementation. The TenantId column will store the tenant ID of this record. The TenantPath column will store the path from the Tenant ID to the root tenant if there is a hierarchical tenant relationship. Database operations on this model will automatically take tenancy into consideration based on the current security context.

// Tenancy is an embedded type for data model. It's responsible for populating TenantPath and check for Tenancy related data
// when crating/updating. Tenancy implements
// - callbacks.BeforeCreateInterface
// - callbacks.BeforeUpdateInterface
// When used as an embedded type, tag `filter` can be used to override default tenancy check behavior:
// - `filter:"w"`: 	create/update/delete are enforced (Default mode)
// - `filter:"rw"`: CRUD operations are all enforced,
//					this mode filters result of any Select/Update/Delete query based on current security context
// - `filter:"-"`: 	filtering is disabled. Note: setting TenantID to in-accessible tenant is still enforced.
//					to disable TenantID value check, use SkipTenancyCheck
// e.g.
// <code>
// type TenancyModel struct {
//		ID         uuid.UUID `gorm:"primaryKey;type:uuid;default:gen_random_uuid();"`
//		Tenancy    `filter:"rw"`
// }
// </code>
type Tenancy struct {
	TenantID   uuid.UUID  `gorm:"type:KeyID;not null"`
	TenantPath TenantPath `gorm:"type:uuid[];index:,type:gin;not null"  json:"-"`
}

Misc

These models are provided as convenient types that can be embedded in application model.

type Audit struct {
	CreatedAt time.Time      `json:"createdAt,omitempty"`
	UpdatedAt time.Time      `json:"updatedAt,omitempty"`
	CreatedBy uuid.UUID      `type:"KeyID;" json:"createdBy,omitempty"`
	UpdatedBy uuid.UUID      `type:"KeyID;" json:"updatedBy,omitempty"`
}

type SoftDelete struct {
	DeletedAt gorm.DeletedAt `gorm:"index" json:"deleteAt,omitempty"`
}

In addition, check the pqx package for common data types such as Duration, Jsonb, TimeArray, UUIDArray

Documentation

Index

• Constants
• Variables
• func ErrorHandlingGormConfigurer() fx.Annotated
• func NewGorm(di gormInitDI) *gorm.DB
• type DataError
  • func NewConstraintViolationError(value interface{}, causes ...interface{}) DataError
  • func NewDataError(code int64, e interface{}, causes ...interface{}) DataError
  • func NewDuplicateKeyError(value interface{}, causes ...interface{}) DataError
  • func NewErrorWithStatusCode(err error, sc int) DataError
  • func NewInternalError(value interface{}, causes ...interface{}) DataError
  • func NewRecordNotFoundError(value interface{}, causes ...interface{}) DataError
• type DataProperties
  • func BindDataProperties(ctx *bootstrap.ApplicationContext) DataProperties
  • func NewDataProperties() *DataProperties
• type DbCreator
• type DbHealthIndicator
  • func (i *DbHealthIndicator) Health(c context.Context, options health.Options) health.Health
  • func (i *DbHealthIndicator) Name() string
• type DefaultGormErrorTranslator
  • func (t DefaultGormErrorTranslator) TranslateWithDB(db *gorm.DB) error
• type ErrorTranslator
  • func NewGormErrorTranslator() ErrorTranslator
  • func NewWebDataErrorTranslator() ErrorTranslator
• type GormConfigurer
• type GormErrorTranslator
• type GormLogger
  • func (l GormLogger) Error(ctx context.Context, s string, i ...interface{})
  • func (l GormLogger) Info(ctx context.Context, s string, i ...interface{})
  • func (l GormLogger) LogMode(level gormlogger.LogLevel) gormlogger.Interface
  • func (l GormLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error)
  • func (l GormLogger) Warn(ctx context.Context, s string, i ...interface{})
• type LoggingProperties
• type TransactionProperties
• type WebDataErrorTranslator
  • func (WebDataErrorTranslator) Order() int
  • func (t WebDataErrorTranslator) Translate(ctx context.Context, err error) error

Constants

const (
	ErrorTranslatorOrderGorm // gorm error -> data error
	ErrorTranslatorOrderData // data error -> data error with status code
)

const (
	ErrorTypeCodeInternal = Reserved + iota<<ErrorTypeOffset
	ErrorTypeCodeNonTransient
	ErrorTypeCodeTransient
	ErrorTypeCodeUncategorizedServerSide
)

All "Type" values are used as mask

const (
	ErrorSubTypeCodeQuery = ErrorTypeCodeNonTransient + iota<<ErrorSubTypeOffset
	ErrorSubTypeCodeApi
	ErrorSubTypeCodeDataRetrieval
	ErrorSubTypeCodeDataIntegrity
	ErrorSubTypeCodeTransaction
	ErrorSubTypeCodeSecurity
)

All "SubType" values are used as mask sub types of ErrorTypeCodeNonTransient

const (
	ErrorSubTypeCodeConcurrency = ErrorTypeCodeTransient + iota<<ErrorSubTypeOffset
	ErrorSubTypeCodeTimeout
	ErrorSubTypeCodeReplica
)

All "SubType" values are used as mask sub types of ErrorTypeCodeTransient

const (
	ErrorCodeInvalidSQL = ErrorSubTypeCodeQuery + iota
	ErrorCodeInvalidPagination
)

ErrorSubTypeCodeQuery

const (
	ErrorCodeInvalidApiUsage = ErrorSubTypeCodeApi + iota
	ErrorCodeUnsupportedCondition
	ErrorCodeUnsupportedOptions
	ErrorCodeInvalidCrudModel
	ErrorCodeInvalidCrudParam
)

ErrorSubTypeCodeApi

const (
	ErrorCodeRecordNotFound = ErrorSubTypeCodeDataRetrieval + iota
	ErrorCodeOrmMapping
	ErrorCodeIncorrectRecordCount
)

ErrorSubTypeCodeDataRetrieval

const (
	ErrorCodeDuplicateKey = ErrorSubTypeCodeDataIntegrity + iota
	ErrorCodeConstraintViolation
	ErrorCodeInvalidSchema
)

ErrorSubTypeCodeDataIntegrity

const (
	ErrorCodeAuthenticationFailed = ErrorSubTypeCodeSecurity + iota
	ErrorCodeFieldOperationDenied
)

ErrorSubTypeCodeSecurity

const (
	ErrorCodePessimisticLocking = ErrorSubTypeCodeConcurrency + iota
	ErrorCodeOptimisticLocking
)

ErrorSubTypeCodeConcurrency

const (
	GormCallbackBeforeCreate = "gorm:before_create"
	GormCallbackAfterCreate  = "gorm:after_create"
	GormCallbackBeforeQuery  = "gorm:query"
	GormCallbackAfterQuery   = "gorm:after_query"
	GormCallbackBeforeUpdate = "gorm:before_update"
	GormCallbackAfterUpdate  = "gorm:after_update"
	GormCallbackBeforeDelete = "gorm:before_delete"
	GormCallbackAfterDelete  = "gorm:after_delete"
	GormCallbackBeforeRow    = "gorm:row"
	GormCallbackAfterRow     = "gorm:row"
	GormCallbackBeforeRaw    = "gorm:raw"
	GormCallbackAfterRaw     = "gorm:raw"
)

const (
	ErrorCodeInternal = ErrorSubTypeCodeInternal + iota
)

ErrorSubTypeCodeInternal

const (
	ErrorCodeInvalidTransaction = ErrorSubTypeCodeTransaction + iota
)

ErrorSubTypeCodeTransaction

const (
	ErrorCodeQueryTimeout = ErrorSubTypeCodeTimeout + iota
)

ErrorSubTypeCodeTimeout

const (
	ErrorCodeReplicaUnavailable = ErrorSubTypeCodeReplica + iota
)

ErrorSubTypeCodeApi

const (
	ErrorSubTypeCodeInternal = ErrorTypeCodeInternal + iota<<ErrorSubTypeOffset
)

All "SubType" values are used as mask sub types of ErrorTypeCodeInternal

const (
	GormConfigurerGroup = "gorm_config"
)

const (
	ManagementPropertiesPrefix = "data"
)

const (
	// Reserved data reserved reserved error range
	Reserved = 0xdb << ReservedOffset
)

Variables

var (
	ErrorCategoryData                = NewErrorCategory(Reserved, errors.New("error type: data"))
	ErrorTypeInternal                = NewErrorType(ErrorTypeCodeInternal, errors.New("error type: internal"))
	ErrorTypeNonTransient            = NewErrorType(ErrorTypeCodeNonTransient, errors.New("error type: non-transient"))
	ErrorTypeTransient               = NewErrorType(ErrorTypeCodeTransient, errors.New("error type: transient"))
	ErrorTypeUnCategorizedServerSide = NewErrorType(ErrorTypeCodeUncategorizedServerSide, errors.New("error type: uncategorized server-side"))

	ErrorSubTypeInternalError = NewErrorSubType(ErrorSubTypeCodeInternal, errors.New("error sub-type: internal"))

	ErrorSubTypeQuery         = NewErrorSubType(ErrorSubTypeCodeQuery, errors.New("error sub-type: query"))
	ErrorSubTypeApi           = NewErrorSubType(ErrorSubTypeCodeApi, errors.New("error sub-type: api"))
	ErrorSubTypeDataRetrieval = NewErrorSubType(ErrorSubTypeCodeDataRetrieval, errors.New("error sub-type: retrieval"))
	ErrorSubTypeDataIntegrity = NewErrorSubType(ErrorSubTypeCodeDataIntegrity, errors.New("error sub-type: integrity"))
	ErrorSubTypeTransaction   = NewErrorSubType(ErrorSubTypeCodeTransaction, errors.New("error sub-type: transaction"))
	ErrorSubTypeSecurity      = NewErrorSubType(ErrorSubTypeCodeSecurity, errors.New("error sub-type: security"))

	ErrorSubTypeConcurrency = NewErrorSubType(ErrorSubTypeCodeConcurrency, errors.New("error sub-type: concurency"))
	ErrorSubTypeTimeout     = NewErrorSubType(ErrorSubTypeCodeTimeout, errors.New("error sub-type: timeout"))
	ErrorSubTypeReplica     = NewErrorSubType(ErrorSubTypeCodeReplica, errors.New("error sub-type: replica"))
)

ErrorTypes, can be used in errors.Is

var (
	ErrorSortByUnknownColumn  = NewDataError(ErrorCodeOrmMapping, "SortBy column unknown")
	ErrorRecordNotFound       = NewDataError(ErrorCodeRecordNotFound, gorm.ErrRecordNotFound)
	ErrorIncorrectRecordCount = NewDataError(ErrorCodeIncorrectRecordCount, "incorrect record count")
	ErrorDuplicateKey         = NewDataError(ErrorCodeDuplicateKey, "duplicate key")
)

Concrete error, can be used in errors.Is for exact match

var (
	GormErrorMapping = map[error]DataError{
		gorm.ErrRecordNotFound:        NewDataError(ErrorCodeRecordNotFound, gorm.ErrRecordNotFound),
		gorm.ErrInvalidTransaction:    NewDataError(ErrorCodeInvalidTransaction, gorm.ErrInvalidTransaction),
		gorm.ErrNotImplemented:        NewDataError(ErrorCodeInvalidApiUsage, gorm.ErrNotImplemented),
		gorm.ErrMissingWhereClause:    NewDataError(ErrorCodeInvalidSQL, gorm.ErrMissingWhereClause),
		gorm.ErrUnsupportedRelation:   NewDataError(ErrorCodeInvalidSchema, gorm.ErrUnsupportedRelation),
		gorm.ErrPrimaryKeyRequired:    NewDataError(ErrorCodeInvalidSQL, gorm.ErrPrimaryKeyRequired),
		gorm.ErrModelValueRequired:    NewDataError(ErrorCodeOrmMapping, gorm.ErrModelValueRequired),
		gorm.ErrInvalidData:           NewDataError(ErrorCodeOrmMapping, gorm.ErrInvalidData),
		gorm.ErrUnsupportedDriver:     NewDataError(ErrorCodeInternal, gorm.ErrUnsupportedDriver),
		gorm.ErrRegistered:            NewDataError(ErrorCodeInternal, gorm.ErrRegistered),
		gorm.ErrInvalidField:          NewDataError(ErrorCodeInvalidSQL, gorm.ErrInvalidField),
		gorm.ErrEmptySlice:            NewDataError(ErrorCodeIncorrectRecordCount, gorm.ErrEmptySlice),
		gorm.ErrDryRunModeUnsupported: NewDataError(ErrorCodeInvalidApiUsage, gorm.ErrDryRunModeUnsupported),
		gorm.ErrInvalidDB:             NewDataError(ErrorCodeInvalidApiUsage, gorm.ErrInvalidDB),
		gorm.ErrInvalidValue:          NewDataError(ErrorCodeInvalidSQL, gorm.ErrInvalidValue),
		gorm.ErrInvalidValueOfLength:  NewDataError(ErrorCodeInvalidSQL, gorm.ErrInvalidValueOfLength),
	}
)

var Module = &bootstrap.Module{
	Name:       "DB",
	Precedence: bootstrap.DatabasePrecedence,
	Options: []fx.Option{
		fx.Provide(BindDataProperties),
		fx.Invoke(registerHealth),
	},
}

Functions

func ErrorHandlingGormConfigurer

func ErrorHandlingGormConfigurer() fx.Annotated

func NewGorm

func NewGorm(di gormInitDI) *gorm.DB

Types

type DataError

type DataError interface {
	error
	NestedError
	Details() interface{}
	WithDetails(interface{}) DataError
	WithMessage(msg string, args ...interface{}) DataError
}

func NewConstraintViolationError

func NewConstraintViolationError(value interface{}, causes ...interface{}) DataError

func NewDataError

func NewDataError(code int64, e interface{}, causes ...interface{}) DataError

func NewDuplicateKeyError

func NewDuplicateKeyError(value interface{}, causes ...interface{}) DataError

func NewErrorWithStatusCode

func NewErrorWithStatusCode(err error, sc int) DataError

func NewInternalError

func NewInternalError(value interface{}, causes ...interface{}) DataError

func NewRecordNotFoundError

func NewRecordNotFoundError(value interface{}, causes ...interface{}) DataError

type DataProperties

type DataProperties struct {
	Logging     LoggingProperties     `json:"logging"`
	Transaction TransactionProperties `json:"transaction"`
}

func BindDataProperties

func BindDataProperties(ctx *bootstrap.ApplicationContext) DataProperties

BindDataProperties create and bind SessionProperties, with a optional prefix

func NewDataProperties

func NewDataProperties() *DataProperties

NewDataProperties create a DataProperties with default values

type DbCreator

type DbCreator interface {
	CreateDatabaseIfNotExist(ctx context.Context, db *gorm.DB) error
}

type DbHealthIndicator

type DbHealthIndicator struct {
	// contains filtered or unexported fields
}

DbHealthIndicator Note: we currently only support one database

func (*DbHealthIndicator) Health

func (i *DbHealthIndicator) Health(c context.Context, options health.Options) health.Health

func (*DbHealthIndicator) Name

func (i *DbHealthIndicator) Name() string

type DefaultGormErrorTranslator

type DefaultGormErrorTranslator struct {
	ErrorTranslator
}

func (DefaultGormErrorTranslator) TranslateWithDB

func (t DefaultGormErrorTranslator) TranslateWithDB(db *gorm.DB) error

type ErrorTranslator

type ErrorTranslator interface {
	order.Ordered
	Translate(ctx context.Context, err error) error
}

ErrorTranslator redefines web.ErrorTranslator and order.Ordered having this redefinition is to break dependency between data and web package

func NewGormErrorTranslator

func NewGormErrorTranslator() ErrorTranslator

func NewWebDataErrorTranslator

func NewWebDataErrorTranslator() ErrorTranslator

type GormConfigurer

type GormConfigurer interface {
	Configure(config *gorm.Config)
}

type GormErrorTranslator

type GormErrorTranslator interface {
	TranslateWithDB(db *gorm.DB) error
}

type GormLogger

type GormLogger struct {
	// contains filtered or unexported fields
}

func (GormLogger) Error

func (l GormLogger) Error(ctx context.Context, s string, i ...interface{})

func (GormLogger) Info

func (l GormLogger) Info(ctx context.Context, s string, i ...interface{})

func (GormLogger) LogMode

func (l GormLogger) LogMode(level gormlogger.LogLevel) gormlogger.Interface

func (GormLogger) Trace

func (l GormLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error)

func (GormLogger) Warn

func (l GormLogger) Warn(ctx context.Context, s string, i ...interface{})

type LoggingProperties

type LoggingProperties struct {
	Level         log.LoggingLevel `json:"level"`
	SlowThreshold utils.Duration   `json:"slow-threshold"`
}

type TransactionProperties

type TransactionProperties struct {
	MaxRetry int `json:"max-retry"`
}

type WebDataErrorTranslator

type WebDataErrorTranslator struct{}

WebDataErrorTranslator implements web.ErrorTranslator

func (WebDataErrorTranslator) Order

func (WebDataErrorTranslator) Order() int

func (WebDataErrorTranslator) Translate

func (t WebDataErrorTranslator) Translate(ctx context.Context, err error) error