Tt

README

Tarantool Adapter and ORM Generator

This package provides automatic schema migration (only for appending new columns, not for changing data types). This package also can be used to generate ORM.

Dependencies

go install github.com/fatih/gomodifytags@latest
go install github.com/kokizzu/replacer@latest

Generated ORM Example

How to create a connection

import "github.com/tarantool/go-tarantool"
import "github.com/kokizzu/gotro/L"

func ConnectTarantool() *tarantool.Connection {
	hostPort := fmt.Sprintf(`%s:%s`,
		TARANTOOL_HOST,
		TARANTOOL_PORT,
	)
	taran, err := tarantool.Connect(hostPort, tarantool.Opts{
		User: TARANTOOL_USER,
		Pass: TARANTOOL_PASS,
	})
	L.PanicIf(err, `tarantool.Connect `+hostPort)
	return taran
}

// then use it like this:
tt := &Tt.Adapter{Connection: ConnectTarantool(), Reconnect: ConnectTarantool}

Usage

1. create a model/ directory inside project
2. create a m[Domain] directory inside project, for example if the domain is authentication, you might want to create mAuth
3. create a [domain]_tables.go something like this:

package mAuth

import "github.com/kokizzu/gotro/D/Tt"

const (
	TableUserss Tt.TableName = `users`
	Id                       = `id`
	Email                    = `email`
	Password                 = `password`
	CreatedBy                = `createdBy`
	CreatedAt                = `createdAt`
	UpdatedBy                = `updatedBy`
	UpdatedAt                = `updatedAt`
	DeletedBy                = `deletedBy`
	DeletedAt                = `deletedAt`
	IsDeleted                = `isDeleted`
	RestoredBy               = `restoredBy`
	RestoredAt               = `restoredAt`
	PasswordSetAt            = `passwordSetAt`
	SecretCode               = `secretCode`
	SecretCodeAt             = `secretCodeAt`
	VerificationSentAt       = `verificationSentAt`
	VerifiedAt               = `verifiedAt`
	LastLoginAt              = `lastLoginAt`
)

const (
	TableSessions Tt.TableName = `sessions`
	SessionToken               = `sessionToken`
	UserId                     = `userId`
	ExpiredAt                  = `expiredAt`
)

var TarantoolTables = map[Tt.TableName]*Tt.TableProp{
	// can only adding fields on back, and must IsNullable: true
	// primary key must be first field and set to Unique: fieldName
	TableUserss: {
		Fields: []Tt.Field{
			{Id, Tt.Unsigned},
			{Email, Tt.String},
			{Password, Tt.String},
			{CreatedAt, Tt.Integer},
			{CreatedBy, Tt.Unsigned},
			{UpdatedAt, Tt.Integer},
			{UpdatedBy, Tt.Unsigned},
			{DeletedAt, Tt.Integer},
			{DeletedBy, Tt.Unsigned},
			{IsDeleted, Tt.Boolean},
			{RestoredAt, Tt.Integer},
			{RestoredBy, Tt.Unsigned},
			{PasswordSetAt, Tt.Integer},
			{SecretCode, Tt.String},
			{SecretCodeAt, Tt.Integer},
			{VerificationSentAt, Tt.Integer},
			{VerifiedAt, Tt.Integer},
			{LastLoginAt, Tt.Integer},
		},
		AutoIncrementId: true,
		Unique2: Email,
		Indexes: []string{IsDeleted, SecretCode},
	},
	TableSessions: {
		Fields: []Tt.Field{
			{SessionToken, Tt.String},
			{UserId, Tt.Unsigned},
			{ExpiredAt, Tt.Integer},
		},
		Unique1: SessionToken,
	},
}

func GenerateORM() {
	Tt.GenerateOrm(TarantoolTables)
}

1. create a [domain]_generator_test.go something like this:

package mAuth

import (
	"testing"
)

//go:generate go test -run=XXX -bench=Benchmark_GenerateOrm

func Benchmark_GenerateOrm(b *testing.B) {
	GenerateORM()
	b.SkipNow()
}

1. run the test to generate new ORM, that would generate rq[Domain]/rq[Domain]__ORM.GEN.go and wc[Domain]/wc[Domain]__ORM.GEN.go file, you might want to create a helper script for that:

#!/usr/bin/env bash

cd ./model
  cat *.go | grep '//go:generate ' | cut -d ' ' -f 2- | bash -x > /tmp/1.log
  
for i in ./m*; do
  if [[ ! -d "$i" ]] ; then continue ; fi
  echo $i
  pushd .
  cd "$i"
  
  # generate ORM
  go test -bench=.
  
  for j in ./*; do 
    echo $j
    if [[ ! -d "$j" ]] ; then continue ; fi
        
    pushd .
    cd "$j" 
    echo `pwd` 
    cat *.go | grep '//go:generate ' | cut -d ' ' -f 2- | bash -x >> /tmp/1.log    
    popd 
    
  done
  
  popd
  
done

1. in your web server engine/domain logic (one that initializes dependencies), create methods to help initialize the buffer, something like this:

type Domain struct {
	Taran     *Tt.Adapter
}

func NewDomain() *Domain {
	d := &Domain{
		Taran: &Tt.Adapter{conf.ConnectTarantool(), conf.ConnectTarantool},
	}
	return d
}

1. last step is just call generated method to manipulate or query, something like this:

func (d *Domain) BusinessLogic1(in *BusinessLogic1_In) (out BusinessLogic1_Out) {
	
	// do something else
	
	user := wcAuth.NewUserMutator(d.Taran)
	user.Email = in.Email
	if !user.FindById() {
		user.Id = id64.UID()
		user.CreatedAt = in.UnixNow()
		if !user.DoInsert() {
			out.SetError(500, `failed to insert user record, db down?`)
			return
		}
	}
	user.SetUpdatedAt(in.UnixNow())
	// do other manipulation
	// use .Set* if you have to call DoUpdateBy*()
	if !user.DoUpdateById() {
		out.SetError(500, `failed to insert user record, db down?`)
		return		
	}
	
}

// or if you only need to read
func (d *Domain) mustLogin(token string, userAgent string, out *ResponseCommon) *conf.Session {
	sess := &conf.Session{}
	if token == `` {
		out.SetError(400, `missing session token`)
		return nil
	}
	if !sess.Decrypt(token, userAgent) {
		out.SetError(400, `invalid session token`) // if got this, possibly wrong userAgent-sessionToken pair
		return nil
	}
	if sess.ExpiredAt <= fastime.UnixNow() {
		out.SetError(400, `token expired`)
		return nil
	}

	session := rqAuth.NewSessions(d.Taran)
	session.SessionToken = token
	if !session.FindBySessionToken() {
		out.SetError(400, `session missing from database, wrong env?`)
		return nil
	}
	if session.ExpiredAt <= fastime.UnixNow() {
		out.SetError(403, `session expired or logged out`)
		return nil
	}
	return sess
}

1. If you need to create an extension method for the ORM, just add a new file on rq[Domain]/anything.go, with a new struct method from generated ORM, something like this:

package rqAuth

import (
	"myProject/conf"
	
	"github.com/kokizzu/gotro/I"
	"github.com/kokizzu/gotro/L"
	"github.com/kokizzu/gotro/X"
	"golang.org/x/crypto/bcrypt"
)

func (s *Users) FindOffsetLimit(offset, limit uint32) (res []*Users) {
	query := `
SELECT ` + s.SqlSelectAllFields() + `
FROM ` + s.SqlTableName() + `
ORDER BY ` + s.SqlId() + `
LIMIT ` + X.ToS(limit) + `
OFFSET ` + X.ToS(offset) // note: for string, use S.Z or S.XSS to prevent SQL injection
	if conf.DEBUG_MODE {
		L.Print(query)
	}
	s.Adapter.QuerySql(query, func(row []any) {
		obj := &Users{}
		obj.FromArray(row)
		obj.CensorFields()
		res = append(res, obj)
	})
	return
}

func (s *Users) CheckPassword(currentPassword string) bool {
	hash := []byte(s.Password)
	pass := []byte(currentPassword)
	err := bcrypt.CompareHashAndPassword(hash, pass)

	return !L.IsError(err, `bcrypt.CompareHashAndPassword`)
}

// call before outputting to client
func (s *Users) CensorFields() {
	s.Password = ``
	s.SecretCode = ``
}

or in wc[Domain]/anything.go if you need to manipulate things

func (p *UsersMutator) SetEncryptPassword(password string) bool {
	pass, err := bcrypt.GenerateFromPassword([]byte(password), 0)
	p.SetPassword(string(pass))
	return !L.IsError(err, `bcrypt.GenerateFromPassword`)
}

1. to initialize automatic migration, just create model/run_migration.go

func RunMigration() {
	L.Print(`run migration..`)
	tt := &Tt.Adapter{Connection: ConnectTarantool(), Reconnect: ConnectTarantool}
	tt.MigrateTables(mAuth.ClickhouseTables)
	// add other tarantool tables to be migrated here
}

then call it on main

func main() {
	model.RunMigration()
}

Documentation

Index

• Constants
• Variables
• func CheckTarantoolTables(tConn *Adapter, tables map[TableName]*TableProp)
• func Connect1(host, port, user, pass string) *tarantool.Connection
• func Descr(args ...any)
• func GenerateOrm(tables map[TableName]*TableProp, withGraphql ...bool)
• func TypeGraphql(field Field) string
• type Adapter
  • func NewAdapter(connectFunc func() *tarantool.Connection) *Adapter
  • func (a *Adapter) CallBoxSpace(funcName string, params A.X) (rows [][]any)
  • func (a *Adapter) CreateSpace(tableName string, engine EngineType) bool
  • func (a *Adapter) DropTable(tableName string) bool
  • func (a *Adapter) ExecBoxSpace(funcName string, params A.X) bool
  • func (a *Adapter) ExecBoxSpaceVerbose(funcName string, params A.X) string
  • func (a *Adapter) ExecSql(query string, parameters ...MSX) map[any]any
  • func (a *Adapter) ExecTarantool(funcName string, params A.X) bool
  • func (a *Adapter) ExecTarantoolVerbose(funcName string, params A.X) string
  • func (a *Adapter) MigrateTables(tables map[TableName]*TableProp)
  • func (a *Adapter) QuerySql(query string, callback func(row []any), parameters ...MSX) []any
  • func (a *Adapter) ReformatTable(tableName string, fields []Field) bool
  • func (a *Adapter) TruncateTable(tableName string) bool
  • func (a *Adapter) UpsertTable(tableName TableName, prop *TableProp) bool
• type DataType
• type EngineType
• type Field
• type Index
• type MSX
• type NullableField
• type QueryMeta
  • func QueryMetaFrom(res *tarantool.Response, err error) QueryMeta
• type TableName
• type TableProp
• type TtDockerTest
  • func (in *TtDockerTest) ConnectCheck(res *dockertest.Resource) (taran *tarantool.Connection, err error)
  • func (in *TtDockerTest) ImageVersion(pool *D.DockerTest, version string) *dockertest.RunOptions
  • func (in *TtDockerTest) SetDefaults(img string)

Constants

const BoxSpacePrefix = `box.space.`

const Engine = `engine`

misc

const IdCol = `id`

const IfNotExists = `if_not_exists`

const NL = "\n"

Variables

var DEBUG = false

var DebugPerf = false

var ErrMiscaonfiguration = errors.New(`misconfiguration`)

var MIGRATION_QUEUE = 1000

var MIGRATION_REC_PER_BATCH = uint64(20_000)

var MIGRATION_THREAD = 8

var Type2TarantoolDefault = map[DataType]string{
	Unsigned: `0`,
	String:   `''`,
	Integer:  `0`,
	Boolean:  `false`,
	Double:   `0`,
	Array:    `[]`,
}

var TypeToConst = map[DataType]string{
	Unsigned: `Tt.Unsigned`,
	String:   `Tt.String`,
	Integer:  `Tt.Integer`,
	Double:   `Tt.Double`,
	Boolean:  `Tt.Boolean`,
	Array:    `Tt.Array`,
}

var TypeToConvertFunc = map[DataType]string{
	Unsigned: `X.ToU`,
	String:   `X.ToS`,
	Integer:  `X.ToI`,
	Double:   `X.ToF`,
	Boolean:  `X.ToBool`,
	Array:    `X.ToArr`,
}

var TypeToGoEmptyValue = map[DataType]string{
	Unsigned: `0`,
	String:   "``",
	Integer:  `0`,
	Double:   `0`,
	Boolean:  `false`,
	Array:    `[]any{}`,
}

var TypeToGoNilValue = map[DataType]string{
	Unsigned: `0`,
	String:   "``",
	Integer:  `0`,
	Double:   `0`,
	Boolean:  `false`,
	Array:    `nil`,
}

var TypeToGoType = map[DataType]string{

	Unsigned: `uint64`,
	String:   `string`,
	Integer:  `int64`,
	Double:   `float64`,
	Boolean:  `bool`,
	Array:    `[]any`,
}

Functions

func CheckTarantoolTables

func CheckTarantoolTables(tConn *Adapter, tables map[TableName]*TableProp)

func Connect1

func Connect1(host, port, user, pass string) *tarantool.Connection

Connect1 is example of connect function to connect on terminal locally, use: tarantoolctl connect user:password@localhost:3301

func Descr

func Descr(args ...any)

func GenerateOrm

func GenerateOrm(tables map[TableName]*TableProp, withGraphql ...bool)

func TypeGraphql

func TypeGraphql(field Field) string

Types

type Adapter

type Adapter struct {
	*tarantool.Connection
	Reconnect func() *tarantool.Connection
}

func NewAdapter

func NewAdapter(connectFunc func() *tarantool.Connection) *Adapter

NewAdapter create new tarantool adapter adapter contains helper methods for schema manipulation and query execution

func (*Adapter) CallBoxSpace

func (a *Adapter) CallBoxSpace(funcName string, params A.X) (rows [][]any)

ignore return value

func (*Adapter) CreateSpace

func (a *Adapter) CreateSpace(tableName string, engine EngineType) bool

func (*Adapter) DropTable

func (a *Adapter) DropTable(tableName string) bool

func (*Adapter) ExecBoxSpace

func (a *Adapter) ExecBoxSpace(funcName string, params A.X) bool

ignore return value

func (*Adapter) ExecBoxSpaceVerbose

func (a *Adapter) ExecBoxSpaceVerbose(funcName string, params A.X) string

func (*Adapter) ExecSql

func (a *Adapter) ExecSql(query string, parameters ...MSX) map[any]any

func (*Adapter) ExecTarantool

func (a *Adapter) ExecTarantool(funcName string, params A.X) bool

ignore return value

func (*Adapter) ExecTarantoolVerbose

func (a *Adapter) ExecTarantoolVerbose(funcName string, params A.X) string

func (*Adapter) MigrateTables

func (a *Adapter) MigrateTables(tables map[TableName]*TableProp)

func (*Adapter) QuerySql

func (a *Adapter) QuerySql(query string, callback func(row []any), parameters ...MSX) []any

func (*Adapter) ReformatTable

func (a *Adapter) ReformatTable(tableName string, fields []Field) bool

func (*Adapter) TruncateTable

func (a *Adapter) TruncateTable(tableName string) bool

func (*Adapter) UpsertTable

func (a *Adapter) UpsertTable(tableName TableName, prop *TableProp) bool

type DataType

type DataType string

types

const (
	Unsigned DataType = `unsigned`
	String   DataType = `string`
	Double   DataType = `double`
	Integer  DataType = `integer`
	Boolean  DataType = `boolean`
	Array    DataType = `array`
)

type EngineType

type EngineType string

const (
	Vinyl EngineType = `vinyl`
	Memtx EngineType = `memtx`
)

type Field

type Field struct {
	Name string   `msgpack:"name"`
	Type DataType `msgpack:"type"`
}

type Index

type Index struct {
	Parts       []string `msgpack:"parts"`
	IfNotExists bool     `msgpack:"if_not_exists"`
	Unique      bool     `msgpack:"unique"`
	Sequence    string   `msgpack:"sequence,omitempty"`
	Type        string   `msgpack:"type,omitempty"`
}

type MSX

type MSX map[string]any

type NullableField

type NullableField struct {
	Name       string   `msgpack:"name"`
	Type       DataType `msgpack:"type"`
	IsNullable bool     `msgpack:"is_nullable"`
}

type QueryMeta

type QueryMeta struct {
	Columns []tarantool.ColumnMetaData
	SqlInfo tarantool.SQLInfo
	Err     string
	Code    uint32
}

func QueryMetaFrom

func QueryMetaFrom(res *tarantool.Response, err error) QueryMeta

type TableName

type TableName string

type TableProp

type TableProp struct {
	Fields []Field

	// indexes
	Unique1 string
	Unique2 string
	Unique3 string
	Uniques []string // multicolumn unique
	Indexes []string
	Spatial string

	Engine          EngineType
	HiddenFields    []string
	AutoIncrementId bool // "id" column will be used to generate sequence, can only be created at beginning
	GenGraphqlType  bool

	// hook
	PreReformatMigrationHook func(*Adapter)
	PreUnique1MigrationHook  func(*Adapter)
	PreUnique2MigrationHook  func(*Adapter)
	PreUnique3MigrationHook  func(*Adapter)
	PreUniquesMigrationHook  func(*Adapter)
	PreSpatialMigrationHook  func(*Adapter)

	AutoCensorFields []string // fields to automatically censor
}

type TtDockerTest

type TtDockerTest struct {
	User     string
	Password string
	Image    string
	Port     string
	// contains filtered or unexported fields
}

func (*TtDockerTest) ConnectCheck

func (in *TtDockerTest) ConnectCheck(res *dockertest.Resource) (taran *tarantool.Connection, err error)

func (*TtDockerTest) ImageVersion

func (in *TtDockerTest) ImageVersion(pool *D.DockerTest, version string) *dockertest.RunOptions

ImageVersion https://hub.docker.com/r/tarantool/tarantool

tarantoolctl connect 3301

func (*TtDockerTest) SetDefaults

func (in *TtDockerTest) SetDefaults(img string)