warehouse

package
Version: v1.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 11, 2022 License: MIT Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source
const RFC3339Micro = "2006-01-02T15:04:05.999999Z07:00"

Variables

View Source
var (
	ErrFileNotFound = errors.New("file not found")
)

Functions

func IndexField

func IndexField(needle WarehouseField, haystack Schema) int

func ValueToString

func ValueToString(val interface{}, isTime bool) string

ValueToString is a common interface method that implementations use to perform value to string conversion

Types

type BaseExportFields

type BaseExportFields struct {
	IndvId                    int64
	UserId                    int64
	SessionId                 int64
	PageId                    int64
	UserCreated               time.Time
	UserAppKey                string
	UserDisplayName           string
	UserEmail                 string
	EventStart                time.Time
	EventType                 string
	EventSubType              string
	EventCustomName           string
	EventTargetText           string
	EventTargetSelector       string
	EventPageOffset           int64
	EventSessionOffset        int64
	EventModFrustrated        int64
	EventModDead              int64
	EventModError             int64
	EventModSuspicious        int64
	EventWebSourceFileUrl     string
	SessionStart              time.Time
	PageStart                 time.Time
	PageDuration              int64
	PageActiveDuration        int64
	PageUrl                   string
	PageRefererUrl            string
	PageIp                    string
	PageLatLong               string
	PageUserAgent             string
	PageBrowser               string
	PageBrowserVersion        string
	PageDevice                string
	PagePlatform              string
	PageOperatingSystem       string
	PageScreenWidth           int64
	PageScreenHeight          int64
	PageViewportWidth         int64
	PageViewportHeight        int64
	PageNumInfos              int64
	PageNumWarnings           int64
	PageNumErrors             int64
	PageClusterId             int64
	PageMaxScrollDepthPercent int64
	LoadDomContentTime        int64
	LoadEventTime             int64
	LoadFirstPaintTime        int64
	ReqUrl                    string
	ReqMethod                 string
	ReqStatus                 int64
	CustomVars                string
}

type BigQuery

type BigQuery struct {
	// contains filtered or unexported fields
}

func NewBigQuery

func NewBigQuery(c *config.BigQueryConfig) *BigQuery

func (*BigQuery) ApplyExportSchema

func (bq *BigQuery) ApplyExportSchema(s Schema) error

func (*BigQuery) GetExportTableColumns

func (bq *BigQuery) GetExportTableColumns() []string

GetExportTableColumns returns a slice of the columns in the existing export table

func (*BigQuery) GetMissingFields

func (bq *BigQuery) GetMissingFields(hauserSchema, bqSchema bigquery.Schema) []*bigquery.FieldSchema

GetMissingFields returns all fields that are present in the hauserSchema, but not in the bqSchema

func (*BigQuery) InitExportTable

func (bq *BigQuery) InitExportTable(s Schema) (bool, error)

func (*BigQuery) LastSyncPoint

func (bq *BigQuery) LastSyncPoint(_ context.Context) (time.Time, error)

func (*BigQuery) LoadToWarehouse

func (bq *BigQuery) LoadToWarehouse(storageRef string, startTime time.Time) error

func (*BigQuery) SaveSyncPoint

func (bq *BigQuery) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*BigQuery) ValueToString

func (bq *BigQuery) ValueToString(val interface{}, isTime bool) string

type Database

type Database interface {
	Syncable
	LoadToWarehouse(storageRef string, start time.Time) error
	ValueToString(val interface{}, isTime bool) string
	GetExportTableColumns() []string

	// InitExportTable should attempt to create the table in the database. If the table doesn't exist, the provided
	// schema should be applied to the table and this function should return `true`, assuming an error didn't occur.
	// If the table existed, this should return false to signal that follow-up schema validation is necessary.
	InitExportTable(Schema) (bool, error)

	// ApplyExportSchema will attempt to update the schema in the database to the provided schema.
	// The provided schema must be compatible, or this will fail. Compatible schemas will have existing columns
	// in the same order as they are currently ordered in the table and can also add new columns to the end.
	ApplyExportSchema(Schema) error
}

type GCSStorage

type GCSStorage struct {
	// contains filtered or unexported fields
}

func NewGCSStorage

func NewGCSStorage(conf *config.GCSConfig, gcsClient *storage.Client) *GCSStorage

func (*GCSStorage) DeleteFile

func (g *GCSStorage) DeleteFile(ctx context.Context, name string) error

func (*GCSStorage) GetFilePrefix added in v1.1.0

func (g *GCSStorage) GetFilePrefix() string

func (*GCSStorage) GetFileReference

func (g *GCSStorage) GetFileReference(name string) string

func (*GCSStorage) LastSyncPoint

func (g *GCSStorage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*GCSStorage) ReadFile

func (g *GCSStorage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*GCSStorage) SaveFile

func (g *GCSStorage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*GCSStorage) SaveSyncPoint

func (g *GCSStorage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type LocalDisk

type LocalDisk struct {
	// contains filtered or unexported fields
}

func NewLocalDisk

func NewLocalDisk(c *config.LocalConfig) *LocalDisk

func (*LocalDisk) DeleteFile

func (w *LocalDisk) DeleteFile(_ context.Context, name string) error

DeleteFile should do nothing for local disk

func (*LocalDisk) GetFilePrefix added in v1.1.0

func (w *LocalDisk) GetFilePrefix() string

func (*LocalDisk) GetFileReference

func (w *LocalDisk) GetFileReference(name string) string

func (*LocalDisk) LastSyncPoint

func (w *LocalDisk) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*LocalDisk) ReadFile

func (w *LocalDisk) ReadFile(_ context.Context, name string) (io.Reader, error)

func (*LocalDisk) SaveFile

func (w *LocalDisk) SaveFile(_ context.Context, name string, reader io.Reader) (string, error)

func (*LocalDisk) SaveSyncPoint

func (w *LocalDisk) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type MobileFields

type MobileFields struct {
	AppName               string
	AppPackageName        string
	AppDeviceModel        string
	AppDeviceVendor       string
	AppVersion            string
	AppOsVersion          string
	AppViewName           string
	EventMobileSourceFile string
}

Mobile Apps fields will not be available for accounts that do not have the feature.

type Redshift

type Redshift struct {
	// contains filtered or unexported fields
}

func NewRedshift

func NewRedshift(c *config.RedshiftConfig) *Redshift

func (*Redshift) ApplyExportSchema

func (rs *Redshift) ApplyExportSchema(newSchema Schema) error

func (*Redshift) CopyInData

func (rs *Redshift) CopyInData(s3file string) error

CopyInData copies data from the given s3File to the export table

func (*Redshift) CreateSyncTable

func (rs *Redshift) CreateSyncTable() error

CreateSyncTable creates a sync table with the hauser sync table schema

func (*Redshift) DeleteExportRecordsAfter

func (rs *Redshift) DeleteExportRecordsAfter(end time.Time) error

func (*Redshift) DoesTableExist

func (rs *Redshift) DoesTableExist(name string) bool

DoesTableExist checks if a table with a given name exists

func (*Redshift) GetExportTableColumns

func (rs *Redshift) GetExportTableColumns() []string

GetExportTableColumns returns all the columns of the export table. It opens a connection and calls getTableColumns

func (*Redshift) InitExportTable

func (rs *Redshift) InitExportTable(schema Schema) (bool, error)

func (*Redshift) LastSyncPoint

func (rs *Redshift) LastSyncPoint(_ context.Context) (time.Time, error)

func (*Redshift) LoadToWarehouse

func (rs *Redshift) LoadToWarehouse(s3obj string, _ time.Time) error

func (*Redshift) MakeRedshiftConnection

func (rs *Redshift) MakeRedshiftConnection() (*sql.DB, error)

func (*Redshift) RemoveOrphanedRecords

func (rs *Redshift) RemoveOrphanedRecords(lastSync pq.NullTime) error

func (*Redshift) SaveSyncPoint

func (rs *Redshift) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*Redshift) ValueToString

func (rs *Redshift) ValueToString(val interface{}, isTime bool) string

type S3Storage

type S3Storage struct {
	// contains filtered or unexported fields
}

func NewS3Storage

func NewS3Storage(conf *config.S3Config) *S3Storage

func (*S3Storage) DeleteFile

func (s *S3Storage) DeleteFile(ctx context.Context, name string) error

func (*S3Storage) GetFilePrefix added in v1.1.0

func (s *S3Storage) GetFilePrefix() string

func (*S3Storage) GetFileReference

func (s *S3Storage) GetFileReference(name string) string

func (*S3Storage) LastSyncPoint

func (s *S3Storage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*S3Storage) ReadFile

func (s *S3Storage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*S3Storage) SaveFile

func (s *S3Storage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*S3Storage) SaveSyncPoint

func (s *S3Storage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Schema

type Schema []WarehouseField

func MakeSchema

func MakeSchema(vals ...interface{}) Schema

func (Schema) Equals

func (s Schema) Equals(other Schema) bool

func (Schema) GetFieldForName

func (s Schema) GetFieldForName(col string) WarehouseField

GetFieldForName takes an existing column name and returns the matching schema field. It performs some conversions between the legacy field names and the new field names.

func (Schema) GetFullStoryFields

func (s Schema) GetFullStoryFields() []string

func (Schema) GetMissingFieldsFor

func (s Schema) GetMissingFieldsFor(b Schema) []WarehouseField

func (Schema) IsCompatibleWith

func (s Schema) IsCompatibleWith(other Schema) bool

func (Schema) ReconcileWithExisting

func (s Schema) ReconcileWithExisting(colNames []string) Schema

ReconcileWithExisting returns a new schema that is compatible with the provided column names. If exported fields are missing from the column list, they are appended to the end.

type Storage

type Storage interface {
	Syncable
	SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)
	ReadFile(ctx context.Context, name string) (io.Reader, error)
	DeleteFile(ctx context.Context, name string) error
	GetFileReference(name string) string
	GetFilePrefix() string
}

type SyncViaStorageMixin

type SyncViaStorageMixin struct {
	// contains filtered or unexported fields
}

SyncViaStorageMixin provides a default implementation for the Syncable interface.

func (SyncViaStorageMixin) LastSyncPoint

func (s SyncViaStorageMixin) LastSyncPoint(ctx context.Context) (time.Time, error)

func (SyncViaStorageMixin) SaveSyncPoint

func (s SyncViaStorageMixin) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Syncable

type Syncable interface {
	LastSyncPoint(ctx context.Context) (time.Time, error)
	SaveSyncPoint(ctx context.Context, endTime time.Time) error
}

type ValueToStringFn

type ValueToStringFn func(val interface{}, isTime bool) string

type WarehouseField

type WarehouseField struct {
	// The name of the field as it exists in the database.
	// By default this will match the `FullStoryFieldName`, but may not match for certain fields
	// if they've been renamed in the export. (e.g. "PageAgent").
	// If the database contains columns that are not part of the FullStory export, this field will still be populated.
	DBName string

	// The name of the field from FullStory.
	// This can be empty if an existing database table contains columns that FullStory does not include
	// in the export. This value, if non-empty, is used as part of the `CreateExport` request in the
	// `DataExportClient`
	FullStoryFieldName string

	// FieldType should be used by each database implementation to specify the datatype
	// for this column. This is only used when creating or modifying a database's schema.
	// If the `FullStoryFieldName` is blank, then this should be nil.
	FieldType reflect.Type
}

WarehouseField contains metadata for a field/column in the warehouse.

func (WarehouseField) IsTime

func (f WarehouseField) IsTime() bool

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL