storage

package
v0.0.0-...-e0c7ee5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 18, 2024 License: Apache-2.0 Imports: 9 Imported by: 0

Documentation

Index

Constants

View Source 
const GcLockID = 100
View Source 
const (
	InsertBatchSize = 1000
)

Variables

This section is empty.

Functions

func CheckStorage 

func CheckStorage(store *gorm.DB) error

CheckStorage verifies storage is valid

Types

type Column 

type Column struct {
	ColumnIdentifier
	DType     string `gorm:"column:data_type;type:varchar(64);comment:'data type like float'"`
	CreatedAt time.Time
	UpdatedAt time.Time
}

type ColumnIdentifier 

type ColumnIdentifier struct {
	ProjectID  string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	TableName  string `gorm:"column:table_name;type:varchar(64);primaryKey;not null"`
	ColumnName string `gorm:"column:column_name;type:varchar(64);primaryKey;not null;"`
}

type ColumnMeta 

type ColumnMeta struct {
	ColumnName string
	DType      string
}

type ColumnPriv 

type ColumnPriv struct {
	ColumnPrivIdentifier
	Priv      string `gorm:"column:priv;type:varchar(256);comment:'priv of column'"`
	CreatedAt time.Time
	UpdatedAt time.Time
}

type ColumnPrivIdentifier 

type ColumnPrivIdentifier struct {
	ProjectID  string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	TableName  string `gorm:"column:table_name;type:varchar(64);primaryKey;not null"`
	ColumnName string `gorm:"column:column_name;type:varchar(64);primaryKey;not null;"`
	DestParty  string `gorm:"column:dest_party;type:varchar(64);primaryKey;not null;"`
}

type Invitation 

type Invitation struct {
	ID               uint64        `gorm:"column:id;primaryKey;comment:'auto generated increment id'"`
	ProjectID        string        `gorm:"column:project_id;type:varchar(64);not null;index:,composite:identifier;comment:'project id'"`
	Name             string        `gorm:"column:name;type:varchar(64);comment:'name'"`
	Description      string        `gorm:"column:desc;type:varchar(64);comment:'description'"`
	Creator          string        `gorm:"column:creator;type:varchar(64);comment:'creator of the project'"`
	ProjectCreatedAt time.Time     `gorm:"column:proj_created_at;comment:'the create time of the project'"`
	Member           string        `gorm:"column:member;type:string;not null;comment:'members, flattened string, like: alice;bob'"`
	ProjectConf      ProjectConfig `gorm:"embedded"`
	Inviter          string        `gorm:"column:inviter;type:varchar(256);index:,composite:identifier;comment:'inviter'"`
	Invitee          string        `gorm:"column:invitee;type:varchar(256);index:,composite:identifier;comment:'invitee'"`
	// 0: default, not decided to accept invitation or not; 1: accepted; 2: rejected; 3: invalid
	Status     int8 `gorm:"column:status;default:0;comment:'accepted'"`
	InviteTime time.Time
	CreatedAt  time.Time
	UpdatedAt  time.Time
}

type Lock 

type Lock struct {
	ID        int8   `gorm:"column:id;primaryKey;uniqueIndex;comment:'lock id';->;<-:create"`
	Owner     string `gorm:"column:owner;type:varchar(64);comment:'lock owner'"`
	UpdatedAt time.Time
}

type Member 

type Member struct {
	ProjectID string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	Member    string `gorm:"column:member;type:varchar(64);primaryKey;not null;comment:'member in the project'"`
}

type MetaManager 

type MetaManager struct {
	// contains filtered or unexported fields
}

func NewMetaManager 

func NewMetaManager(db *gorm.DB, ps bool) *MetaManager

func (*MetaManager) Bootstrap 

func (manager *MetaManager) Bootstrap() error

Bootstrap init db

func (*MetaManager) CheckIdCanceled 

func (m *MetaManager) CheckIdCanceled(ids []string) ([]string, error)

func (*MetaManager) ClearExpiredResults 

func (m *MetaManager) ClearExpiredResults(ttl time.Duration) error

func (*MetaManager) ClearSessionResult 

func (m *MetaManager) ClearSessionResult(id string) (err error)

func (*MetaManager) CreateMetaTransaction 

func (manager *MetaManager) CreateMetaTransaction() *MetaTransaction

create a new MetaTransaction for every request and call Finish when you finish all your actions

func (*MetaManager) DropTables 

func (manager *MetaManager) DropTables() error

drop db for tests

func (*MetaManager) ExecInMetaTransaction 

func (manager *MetaManager) ExecInMetaTransaction(fn func(*MetaTransaction) error) error

func (*MetaManager) GetSessionInfo 

func (m *MetaManager) GetSessionInfo(id string) (SessionInfo, error)

func (*MetaManager) GetSessionResult 

func (m *MetaManager) GetSessionResult(id string) (SessionResult, error)

func (*MetaManager) HoldGcLock 

func (m *MetaManager) HoldGcLock(owner string, ttl time.Duration) error

NOTE: 1. FirstOrCreateGcLock must be called once before call HoldGcLock

  1. lock will expired after ttl, please make sure ttl is long enough for finished tasks

func (*MetaManager) InitGcLockIfNecessary 

func (m *MetaManager) InitGcLockIfNecessary() error

func (*MetaManager) NeedBootstrap 

func (manager *MetaManager) NeedBootstrap() bool

NeedBootstrap checks if the store is empty

func (*MetaManager) SetSessionInfo 

func (m *MetaManager) SetSessionInfo(info SessionInfo) error

func (*MetaManager) SetSessionResult 

func (m *MetaManager) SetSessionResult(sr SessionResult) (err error)

NOTE: check SessionInfo exist before calling

type MetaTransaction 

type MetaTransaction struct {
	// contains filtered or unexported fields
}

func AddExclusiveLock 

func AddExclusiveLock(txn *MetaTransaction) *MetaTransaction

func AddShareLock 

func AddShareLock(txn *MetaTransaction) *MetaTransaction

func (*MetaTransaction) AddInvitations 

func (t *MetaTransaction) AddInvitations(invitations []Invitation) error

duplicated invitations are permitted

func (*MetaTransaction) AddProjectMembers 

func (t *MetaTransaction) AddProjectMembers(members []Member) error

func (*MetaTransaction) AddTable 

func (t *MetaTransaction) AddTable(table TableMeta) error

func (*MetaTransaction) ArchiveProject 

func (t *MetaTransaction) ArchiveProject(projectID string) error

archive project fail if project not exists or other reasons

func (*MetaTransaction) CreateProject 

func (t *MetaTransaction) CreateProject(project Project) error

return err, if project exists

func (*MetaTransaction) DropTable 

func (t *MetaTransaction) DropTable(table TableIdentifier) error

DropTable will drop columns and table

func (*MetaTransaction) DropTableColumns 

func (t *MetaTransaction) DropTableColumns(table TableIdentifier) error

func (*MetaTransaction) Finish 

func (t *MetaTransaction) Finish(err error) error

Automatic rollback will occur if a deadlock or timeout occurs.

func (*MetaTransaction) GetInvitationsBy 

func (t *MetaTransaction) GetInvitationsBy(invitation Invitation, selectUnDecidedStatus bool) (invitations []Invitation, err error)

Note: because undecided status is zero, gorm will ignore it, so if select undecided invitations, set selectUnDecidedStatus true

func (*MetaTransaction) GetProject 

func (t *MetaTransaction) GetProject(projectID string) (Project, error)

func (*MetaTransaction) GetProjectAndMembers 

func (t *MetaTransaction) GetProjectAndMembers(projectID string) (projectAndMembers ProjectWithMember, err error)

func (*MetaTransaction) GetProjectMembers 

func (t *MetaTransaction) GetProjectMembers(projectID string) ([]string, error)

func (*MetaTransaction) GetProjectMeta 

func (t *MetaTransaction) GetProjectMeta(projectID string, tableNames []string, cclDestParties []string, owner string) (*ProjectMeta, error)

return all tables and ccls for the given project if tableNames and cclDestParties are nil

func (*MetaTransaction) GetTableMetasByTableNames 

func (t *MetaTransaction) GetTableMetasByTableNames(projectID string, tableNames []string) (tableMetas []TableMeta, notFoundTables []string, err error)

if len(tableNames) == 0 return all tables return err if ANY table DOESN'T exist

func (*MetaTransaction) GetTables 

func (t *MetaTransaction) GetTables(projectID string, tableNames []string) (tables []Table, allTableExist bool, err error)

func (*MetaTransaction) GetUnhandledInvitation 

func (t *MetaTransaction) GetUnhandledInvitation(projectID, inviter, invitee string) (Invitation, error)

func (*MetaTransaction) GetUnhandledInvitationWithID 

func (t *MetaTransaction) GetUnhandledInvitationWithID(invitationID uint64) (Invitation, error)

func (*MetaTransaction) GrantColumnConstraints 

func (t *MetaTransaction) GrantColumnConstraints(privs []ColumnPriv) error

func (*MetaTransaction) ListColumnConstraints 

func (t *MetaTransaction) ListColumnConstraints(projectID string, tableNames []string, destParties []string) ([]ColumnPriv, error)

func (*MetaTransaction) ListDedupTableOwners 

func (t *MetaTransaction) ListDedupTableOwners(projectID string, tableNames []string) ([]string, error)

func (*MetaTransaction) ListInvitations 

func (t *MetaTransaction) ListInvitations() ([]Invitation, error)

func (*MetaTransaction) ListProjects 

func (t *MetaTransaction) ListProjects(projectIDs []string) ([]ProjectWithMember, error)

func (*MetaTransaction) ModifyInvitationStatus 

func (t *MetaTransaction) ModifyInvitationStatus(invitationID uint64, status pb.InvitationStatus) error

NOTE: invitation id may be zero don't use Where(&Invitation{ID: id})

func (*MetaTransaction) RevokeColumnConstraints 

func (t *MetaTransaction) RevokeColumnConstraints(privIdentifiers []ColumnPrivIdentifier) error

use when ccl changed

func (*MetaTransaction) SetInvitationInvalidByID 

func (t *MetaTransaction) SetInvitationInvalidByID(invitationID uint64) error

func (*MetaTransaction) SetUnhandledInvitationsInvalid 

func (t *MetaTransaction) SetUnhandledInvitationsInvalid(projectID, inviter, invitee string) error

func (*MetaTransaction) UpdateProject 

func (t *MetaTransaction) UpdateProject(proj Project) error

update project fail if project not exists or other reasons

type Project 

type Project struct {
	// ->;<-:create means read and create
	// id can't be modified
	ID          string        `gorm:"column:id;type:varchar(64);primaryKey;uniqueIndex:;comment:'unique id';->;<-:create"`
	Name        string        `gorm:"column:name;type:varchar(64);not null;comment:'project name'"`
	Description string        `gorm:"column:desc;type:varchar(64);comment:'description'"`
	Creator     string        `gorm:"column:creator;type:varchar(64);comment:'creator of the project'"`
	Archived    bool          `gorm:"column:archived;comment:'if archived is true, whole project can't be modified'"`
	ProjectConf ProjectConfig `gorm:"embedded"`
	CreatedAt   time.Time
	UpdatedAt   time.Time
}

type ProjectConfig 

type ProjectConfig struct {
	// in json format
	SpuConf string `gorm:"column:spu_conf;comment:'description'"`
}

type ProjectMeta 

type ProjectMeta struct {
	// TODO: add project info later
	// proj   Project
	Tables []TableMeta
	CCLs   []ColumnPriv
}

type ProjectWithMember 

type ProjectWithMember struct {
	Proj    Project
	Members []string
}

type SessionInfo 

type SessionInfo struct {
	SessionID string `gorm:"column:session_id;type:varchar(64);primaryKey;uniqueIndex:;comment:'unique session id';->;<-:create"`
	// 0: default, running; 1: finished; 2: canceled
	Status        int8   `gorm:"column:status;default:0;comment:'session status'"`
	TableChecksum []byte `gorm:"column:table_checksum;type:varbinary(256);comment:'table checksum for self party'"`
	CCLChecksum   []byte `gorm:"column:ccl_checksum;type:varbinary(256);comment:'ccl checksum for self party'"`
	EngineUrl     string `gorm:"column:engine_url;type:varchar(256);comment:'url for engine to communicate with peer engine'"`
	JobInfo       []byte `gorm:"column:job_info;type:bytes;comment:'serialized job info to specify task in engine'"`
	WorkParties   string `gorm:"column:work_parties;type:string;not null;comment:'parties involved, flattened string, like: alice;bob'"`
	OutputNames   string `gorm:"column:output_names;type:string;comment:'output column names, flattened string, like: col1,col2'"`
	Warning       []byte `gorm:"column:warning;type:bytes;comment:'warning infos, serialized from pb.Warning'"`
	CreatedAt     time.Time
	UpdatedAt     time.Time
}

SessionInfo and SessionResult ares used to support broker cluster mode

type SessionResult 

type SessionResult struct {
	SessionID string `gorm:"column:session_id;type:varchar(64);primaryKey;uniqueIndex:;comment:'unique session id';->;<-:create"`
	Result    []byte `gorm:"column:result;type:bytes;comment:'query result, serialized from protobuf message'"`
	CreatedAt time.Time
	UpdatedAt time.Time
}

type SessionStatus 

type SessionStatus int8
const (
	SessionRunning  SessionStatus = 0
	SessionFinished SessionStatus = 1
	SessionCanceled SessionStatus = 2
)

type Table 

type Table struct {
	TableIdentifier
	RefTable string `gorm:"column:ref_table;type:varchar(128);comment:'ref table'"`
	DBType   string `gorm:"column:db_type;type:varchar(64);comment:'database type like MYSQL'"`
	Owner    string `gorm:"column:owner;comment:'ref table'"`
	// view
	IsView       bool   `gorm:"column:is_view;comment:'this table is a view'"`
	SelectString string `gorm:"column:select_string;comment:'the internal select query in string format, the field is valid only when IsView is true'"`

	CreatedAt time.Time
	UpdatedAt time.Time
}

type TableIdentifier 

type TableIdentifier struct {
	ProjectID string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	TableName string `gorm:"column:table_name;type:varchar(64);primaryKey;not null"`
}

type TableMeta 

type TableMeta struct {
	Table   Table
	Columns []ColumnMeta
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.