metadata

Documentation

Index

• func ConnectToDB(dbURL string, maxConnections int) (*sql.DB, error)
• type Backend
  • func NewPostgresLoader(cfg *PGConfig, lChecker loadChecker, versions versions.Getter) (Backend, error)
• type EventStats
• type Load
• type LoadManifest
• type PGConfig
• type PendingLoadStats
• type PendingLoadType
• type Reader
  • func NewPostgresReader(cfg *PGConfig, versions versions.Getter) (Reader, error)
• type Storer
  • func NewPostgresStorer(cfg *PGConfig) (Storer, error)

Functions

func ConnectToDB

func ConnectToDB(dbURL string, maxConnections int) (*sql.DB, error)

ConnectToDB opens a postgres connection to the db with max open connections

Types

type Backend

type Backend interface {
	Storer
	Reader
	LoadReady() chan *LoadManifest
	LoadError(manifestUUID, loadError string)
	LoadDone(manifestUUID string, tableName string)
	GetLastLoads() map[string]time.Time
}

Backend specifies the interface for load state

func NewPostgresLoader

func NewPostgresLoader(cfg *PGConfig, lChecker loadChecker, versions versions.Getter) (Backend, error)

NewPostgresLoader configures a new postgres backend for loading (or storing) At backend configuration, we set a max number of tsvs for a table and max count of tsvs before a load is triggered.

type EventStats

type EventStats struct {
	Event string
	Count int64
	MinTS time.Time
}

EventStats defines a set of statistics recorded for a particular event.

type Load

type Load scoop_protocol.RowCopyRequest

Load represents a file that needs to be loaded

type LoadManifest

type LoadManifest struct {
	Loads     []Load
	TableName string
	UUID      string
}

LoadManifest represents a set of files that needs to be loaded

type PGConfig

type PGConfig struct {
	DatabaseURL      string
	LoadAgeTrigger   time.Duration
	LoadCountTrigger int
	MaxConnections   int
}

PGConfig stores configuration for postgres

type PendingLoadStats

type PendingLoadStats struct {
	Type  PendingLoadType
	Stats []*EventStats
}

PendingLoadStats stores aggregated stats for events in a particular type of pending load group.

type PendingLoadType

type PendingLoadType string

PendingLoadType specifies a particular reason for events to be pending load.

const (
	// PendingInQueue are loads waiting in active queue.
	PendingInQueue PendingLoadType = "in_queue"

	// PendingStale are loads retried a maximum amount of times.
	PendingStale PendingLoadType = "stale"

	// PendingMigration are loads blocked on version migration.
	PendingMigration PendingLoadType = "pending_migration"
)

type Reader

type Reader interface {
	Versions() (map[string]int, error)
	PingDB() error
	TSVVersionExists(table string, version int) (bool, error)
	ForceLoad(table string, requester string) error
	StatsForPendingLoads() ([]*PendingLoadStats, error)
	IsForceLoadRequested(table string) (bool, error)
}

Reader specifies the interface for Backend read/write operations

func NewPostgresReader

func NewPostgresReader(cfg *PGConfig, versions versions.Getter) (Reader, error)

NewPostgresReader configures a new postgres backend for reading only

type Storer

type Storer interface {
	InsertLoad(load *Load) error
	ListDistinctTables() ([]string, error)
	Close()
}

Storer specifies recording loads in the db

func NewPostgresStorer

func NewPostgresStorer(cfg *PGConfig) (Storer, error)

NewPostgresStorer configures a new postgres backend for storing only