datastore

package
v0.0.0-...-dc8f43e Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 13, 2025 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Overview

Package datastore provides a unified interface for multiple database backends supporting Git object storage, metadata management, and transactional operations.

Index

Constants

View Source 
const (
	TypeMemory   = "memory"
	TypeSQLite   = "sqlite"
	TypeBadger   = "badger"
	TypePostgres = "postgres"
	TypeMongoDB  = "mongodb"
	TypeRedis    = "redis"
	TypeCustom   = "custom"
	TypeBolt     = "bolt"
)

DatabaseType constants

Variables

View Source 
var (
	ErrNotFound          = errors.New("not found")
	ErrAlreadyExists     = errors.New("already exists")
	ErrInvalidData       = errors.New("invalid data")
	ErrTransactionFailed = errors.New("transaction failed")
	ErrNotImplemented    = errors.New("not implemented")
	ErrConnectionFailed  = errors.New("connection failed")
)

Common errors

Functions

func BenchmarkStorageOperations 

func BenchmarkStorageOperations(b *testing.B, store DataStore, storeName string)

BenchmarkStorageOperations provides standardized benchmarks

func ChaosTestSuite 

func ChaosTestSuite(t *testing.T, createStore func() DataStore)

ChaosTestSuite runs chaos engineering tests to verify system resilience

func FuzzTestSuite 

func FuzzTestSuite(t *testing.T, store DataStore)

FuzzTestSuite runs property-based and fuzzing tests

func ListTypes 

func ListTypes() []string

ListTypes returns all registered datastore types

func PerformanceTestSuite 

func PerformanceTestSuite(t *testing.T, stores map[string]DataStore)

PerformanceTestSuite runs comprehensive performance and benchmark tests

func Register 

func Register(name string, factory Factory)

Register registers a new datastore factory

func SecurityTestSuite 

func SecurityTestSuite(t *testing.T, store DataStore)

SecurityTestSuite runs comprehensive security tests

func ValidateStressTestResults 

func ValidateStressTestResults(t *testing.T, metrics *StressTestMetrics, config StressTestConfig)

ValidateStressTestResults checks if stress test results are acceptable

Types

type AuditEvent 

type AuditEvent struct {
	ID         string                 `json:"id" db:"id"`
	Timestamp  time.Time              `json:"timestamp" db:"timestamp"`
	UserID     string                 `json:"user_id" db:"user_id"`
	Username   string                 `json:"username" db:"username"`
	Action     string                 `json:"action" db:"action"`
	Resource   string                 `json:"resource" db:"resource"`
	ResourceID string                 `json:"resource_id" db:"resource_id"`
	Details    map[string]interface{} `json:"details" db:"details"`
	IPAddress  string                 `json:"ip_address" db:"ip_address"`
	UserAgent  string                 `json:"user_agent" db:"user_agent"`
	Success    bool                   `json:"success" db:"success"`
	ErrorMsg   string                 `json:"error_msg,omitempty" db:"error_msg"`
}

AuditEvent represents an auditable action

type Batch 

type Batch interface {
	Put(key string, value []byte)
	Delete(key string)
	Clear()
	Size() int
	Execute() error
}

Batch represents a batch of operations

type CacheConfig 

type CacheConfig struct {
	Enabled  bool          `yaml:"enabled" json:"enabled"`
	Type     string        `yaml:"type" json:"type"` // "memory", "redis"
	Size     int64         `yaml:"size" json:"size"` // Max cache size in bytes
	TTL      time.Duration `yaml:"ttl" json:"ttl"`
	Eviction string        `yaml:"eviction" json:"eviction"` // "lru", "lfu", "fifo"
}

CacheConfig configures caching layer

type Config 

type Config struct {
	Type       string                 `yaml:"type" json:"type"`
	Connection string                 `yaml:"connection" json:"connection"`
	Options    map[string]interface{} `yaml:"options" json:"options"`

	// Advanced configurations
	Cache       *CacheConfig       `yaml:"cache" json:"cache"`
	Sharding    *ShardingConfig    `yaml:"sharding" json:"sharding"`
	Replication *ReplicationConfig `yaml:"replication" json:"replication"`

	// Connection pool settings
	MaxConnections     int           `yaml:"max_connections" json:"max_connections"`
	MaxIdleConnections int           `yaml:"max_idle_connections" json:"max_idle_connections"`
	ConnectionTimeout  time.Duration `yaml:"connection_timeout" json:"connection_timeout"`

	// Performance settings
	BatchSize         int  `yaml:"batch_size" json:"batch_size"`
	EnableCompression bool `yaml:"enable_compression" json:"enable_compression"`
	EnableEncryption  bool `yaml:"enable_encryption" json:"enable_encryption"`
}

Config represents the configuration for a datastore

func DefaultConfig 

func DefaultConfig(dbType string) Config

DefaultConfig returns default configuration for a database type

func (*Config) GetBoolOption 

func (c *Config) GetBoolOption(key string, defaultValue bool) bool

GetBoolOption retrieves a boolean option

func (*Config) GetDurationOption 

func (c *Config) GetDurationOption(key string, defaultValue time.Duration) time.Duration

GetDurationOption retrieves a duration option

func (*Config) GetIntOption 

func (c *Config) GetIntOption(key string, defaultValue int) int

GetIntOption retrieves an integer option

func (*Config) GetOption 

func (c *Config) GetOption(key string, defaultValue interface{}) interface{}

GetOption retrieves a typed option value

func (*Config) GetStringOption 

func (c *Config) GetStringOption(key string, defaultValue string) string

GetStringOption retrieves a string option

func (*Config) Validate 

func (c *Config) Validate() error

Validate checks if the configuration is valid

type DataStore 

type DataStore interface {
	// Lifecycle management
	Initialize(config Config) error
	Close() error
	HealthCheck(ctx context.Context) error

	// Get specific stores
	ObjectStore() ObjectStore
	MetadataStore() MetadataStore

	// Transaction support
	BeginTx(ctx context.Context, opts *TxOptions) (Transaction, error)

	// Metrics and monitoring
	GetMetrics() Metrics

	// Backend information
	Type() string
	Info() map[string]interface{}
}

DataStore is the main interface that all database implementations must satisfy. It combines object storage, metadata management, and transaction support.

func Create 

func Create(config Config) (DataStore, error)

Create creates a new datastore instance

func CreateFromEnv 

func CreateFromEnv() (DataStore, error)

CreateFromEnv creates a datastore from environment variables

func CreateFromJSON 

func CreateFromJSON(jsonConfig string) (DataStore, error)

CreateFromJSON creates a datastore from JSON configuration

func CreateFromYAML 

func CreateFromYAML(yamlConfig string) (DataStore, error)

CreateFromYAML creates a datastore from YAML configuration

type EventFilter 

type EventFilter struct {
	UserID     string
	Action     string
	Resource   string
	ResourceID string
	After      *time.Time
	Before     *time.Time
	Success    *bool
	Limit      int
	Offset     int
	OrderBy    string
}

EventFilter filters audit event queries

type Factory 

type Factory func(config Config) (DataStore, error)

Factory is a function that creates a DataStore instance

type IsolationLevel 

type IsolationLevel int

IsolationLevel defines transaction isolation levels 

const (
	IsolationDefault IsolationLevel = iota
	IsolationReadUncommitted
	IsolationReadCommitted
	IsolationRepeatableRead
	IsolationSerializable
)

type Iterator 

type Iterator interface {
	Next() bool
	Key() string
	Value() []byte
	Error() error
	Close() error
}

Iterator provides a way to iterate over results

type MetadataStore 

type MetadataStore interface {
	// Repository management
	SaveRepository(repo *Repository) error
	GetRepository(id string) (*Repository, error)
	ListRepositories(filter RepositoryFilter) ([]*Repository, error)
	DeleteRepository(id string) error
	UpdateRepository(id string, updates map[string]interface{}) error

	// User management
	SaveUser(user *User) error
	GetUser(id string) (*User, error)
	GetUserByUsername(username string) (*User, error)
	ListUsers(filter UserFilter) ([]*User, error)
	DeleteUser(id string) error
	UpdateUser(id string, updates map[string]interface{}) error

	// Reference management (branches, tags)
	SaveRef(repoID string, ref *Reference) error
	GetRef(repoID string, name string) (*Reference, error)
	ListRefs(repoID string, refType RefType) ([]*Reference, error)
	DeleteRef(repoID string, name string) error
	UpdateRef(repoID string, name string, newHash string) error

	// Audit logging
	LogEvent(event *AuditEvent) error
	QueryEvents(filter EventFilter) ([]*AuditEvent, error)
	CountEvents(filter EventFilter) (int64, error)

	// Configuration
	GetConfig(key string) (string, error)
	SetConfig(key string, value string) error
	GetAllConfig() (map[string]string, error)
	DeleteConfig(key string) error
}

MetadataStore handles repository metadata, users, configuration, etc.

type Metrics 

type Metrics struct {
	// Operation counts
	Reads   int64 `json:"reads"`
	Writes  int64 `json:"writes"`
	Deletes int64 `json:"deletes"`

	// Performance metrics
	AvgReadLatency  time.Duration `json:"avg_read_latency"`
	AvgWriteLatency time.Duration `json:"avg_write_latency"`

	// Storage metrics
	ObjectCount int64 `json:"object_count"`
	StorageSize int64 `json:"storage_size"`

	// Connection metrics
	ActiveConnections int   `json:"active_connections"`
	TotalConnections  int64 `json:"total_connections"`

	// Error metrics
	ErrorCount int64  `json:"error_count"`
	LastError  string `json:"last_error,omitempty"`

	// Uptime
	StartTime time.Time     `json:"start_time"`
	Uptime    time.Duration `json:"uptime"`
}

Metrics provides performance and usage metrics

type MultiStore 

type MultiStore struct {
	// contains filtered or unexported fields
}

MultiStore allows using different stores for different data types

func NewMultiStore 

func NewMultiStore(objectsConfig, metadataConfig, cacheConfig Config) (*MultiStore, error)

NewMultiStore creates a multi-store configuration

type ObjectStore 

type ObjectStore interface {
	// Basic operations
	GetObject(hash string) ([]byte, error)
	PutObject(hash string, data []byte) error
	DeleteObject(hash string) error
	HasObject(hash string) (bool, error)

	// Listing and iteration
	ListObjects(prefix string, limit int) ([]string, error)
	IterateObjects(prefix string, fn func(hash string, data []byte) error) error

	// Batch operations for performance
	GetObjects(hashes []string) (map[string][]byte, error)
	PutObjects(objects map[string][]byte) error
	DeleteObjects(hashes []string) error

	// Size and statistics
	GetObjectSize(hash string) (int64, error)
	CountObjects() (int64, error)
	GetStorageSize() (int64, error)
}

ObjectStore handles Git object storage (blobs, trees, commits, tags)

type PerformanceMetrics 

type PerformanceMetrics struct {
	StoreName           string
	TotalOperations     int64
	OperationsPerSecond float64
	AverageLatency      time.Duration
	P95Latency          time.Duration
	P99Latency          time.Duration
	MinLatency          time.Duration
	MaxLatency          time.Duration
	MemoryUsed          int64
	ErrorCount          int64
	Latencies           []time.Duration
}

PerformanceMetrics tracks detailed performance statistics

type RefType 

type RefType string

RefType defines the type of reference 

const (
	RefTypeBranch RefType = "branch"
	RefTypeTag    RefType = "tag"
)

type Reference 

type Reference struct {
	Name      string    `json:"name" db:"name"`
	Hash      string    `json:"hash" db:"hash"`
	Type      RefType   `json:"type" db:"type"`
	UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
	UpdatedBy string    `json:"updated_by" db:"updated_by"`
}

Reference represents a Git reference (branch or tag)

type Registry 

type Registry struct {
	// contains filtered or unexported fields
}

Registry manages datastore factories

func (*Registry) Create 

func (r *Registry) Create(config Config) (DataStore, error)

Create creates a new datastore using the registered factory

func (*Registry) ListTypes 

func (r *Registry) ListTypes() []string

ListTypes returns all registered datastore types

func (*Registry) Register 

func (r *Registry) Register(name string, factory Factory)

Register registers a factory for a datastore type

type ReplicationConfig 

type ReplicationConfig struct {
	Enabled      bool     `yaml:"enabled" json:"enabled"`
	Mode         string   `yaml:"mode" json:"mode"` // "master-slave", "multi-master"
	Replicas     []string `yaml:"replicas" json:"replicas"`
	SyncMode     string   `yaml:"sync_mode" json:"sync_mode"` // "async", "sync", "semi-sync"
	FailoverAuto bool     `yaml:"failover_auto" json:"failover_auto"`
}

ReplicationConfig configures data replication

type Repository 

type Repository struct {
	ID          string                 `json:"id" db:"id"`
	Name        string                 `json:"name" db:"name"`
	Description string                 `json:"description" db:"description"`
	Path        string                 `json:"path" db:"path"`
	IsPrivate   bool                   `json:"is_private" db:"is_private"`
	CreatedAt   time.Time              `json:"created_at" db:"created_at"`
	UpdatedAt   time.Time              `json:"updated_at" db:"updated_at"`
	Metadata    map[string]interface{} `json:"metadata" db:"metadata"`

	// Statistics
	Size        int64 `json:"size" db:"size"`
	CommitCount int64 `json:"commit_count" db:"commit_count"`
	BranchCount int   `json:"branch_count" db:"branch_count"`
}

Repository represents a Git repository's metadata

type RepositoryFilter 

type RepositoryFilter struct {
	IDs           []string
	Name          string
	IsPrivate     *bool
	CreatedAfter  *time.Time
	CreatedBefore *time.Time
	Limit         int
	Offset        int
	OrderBy       string
}

RepositoryFilter filters repository queries

type ShardingConfig 

type ShardingConfig struct {
	Enabled      bool     `yaml:"enabled" json:"enabled"`
	ShardCount   int      `yaml:"shard_count" json:"shard_count"`
	ShardKey     string   `yaml:"shard_key" json:"shard_key"`
	ShardServers []string `yaml:"shard_servers" json:"shard_servers"`
}

ShardingConfig configures data sharding

type StressTestConfig 

type StressTestConfig struct {
	Duration           time.Duration
	NumWorkers         int
	OperationsPerSec   int
	MaxObjectSize      int
	MinObjectSize      int
	ReadWriteRatio     float64 // 0.0 = all writes, 1.0 = all reads
	TransactionPercent float64 // percentage of operations in transactions
}

StressTestConfig defines parameters for stress testing

func DefaultStressConfig 

func DefaultStressConfig() StressTestConfig

DefaultStressConfig returns default stress test configuration

type StressTestMetrics 

type StressTestMetrics struct {
	TotalOperations     int64
	SuccessfulOps       int64
	FailedOps           int64
	ReadOps             int64
	WriteOps            int64
	DeleteOps           int64
	TransactionOps      int64
	AverageLatency      time.Duration
	MaxLatency          time.Duration
	MinLatency          time.Duration
	MemoryUsagePeak     int64
	ErrorsByType        map[string]int64
	ThroughputOpsPerSec float64
}

StressTestMetrics tracks stress test results

func StressTestSuite 

func StressTestSuite(t *testing.T, store DataStore, config StressTestConfig) *StressTestMetrics

StressTestSuite runs comprehensive stress tests

type Transaction 

type Transaction interface {
	// Transaction control
	Commit() error
	Rollback() error

	// Include all ObjectStore operations
	ObjectStore

	// Include all MetadataStore operations
	MetadataStore
}

Transaction represents a database transaction with all operations

type TxOptions 

type TxOptions struct {
	Isolation IsolationLevel
	ReadOnly  bool
	Timeout   time.Duration
}

TxOptions configures transaction behavior

type User 

type User struct {
	ID          string                 `json:"id" db:"id"`
	Username    string                 `json:"username" db:"username"`
	Email       string                 `json:"email" db:"email"`
	FullName    string                 `json:"full_name" db:"full_name"`
	IsActive    bool                   `json:"is_active" db:"is_active"`
	IsAdmin     bool                   `json:"is_admin" db:"is_admin"`
	CreatedAt   time.Time              `json:"created_at" db:"created_at"`
	UpdatedAt   time.Time              `json:"updated_at" db:"updated_at"`
	LastLoginAt *time.Time             `json:"last_login_at" db:"last_login_at"`
	Metadata    map[string]interface{} `json:"metadata" db:"metadata"`
}

User represents a system user

type UserFilter 

type UserFilter struct {
	IDs      []string
	Username string
	Email    string
	IsActive *bool
	IsAdmin  *bool
	Limit    int
	Offset   int
	OrderBy  string
}

UserFilter filters user queries

Source Files

View all Source files

Directories

Path Synopsis
Package badger implements a BadgerDB-backed datastore for govc.
 Package badger implements a BadgerDB-backed datastore for govc.
Package memory implements an in-memory datastore for govc.
 Package memory implements an in-memory datastore for govc.
Package postgres implements a PostgreSQL-backed datastore for govc.
 Package postgres implements a PostgreSQL-backed datastore for govc.
Package sqlite implements a SQLite-backed datastore for govc.
 Package sqlite implements a SQLite-backed datastore for govc.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.