warehouse

package
v1.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 11, 2022 License: MIT Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const RFC3339Micro = "2006-01-02T15:04:05.999999Z07:00"

Variables

View Source 
var (
	ErrFileNotFound = errors.New("file not found")
)

Functions

func IndexField 

func IndexField(needle WarehouseField, haystack Schema) int

func ValueToString 

func ValueToString(val interface{}, isTime bool) string

ValueToString is a common interface method that implementations use to perform value to string conversion

Types

type BaseExportFields 

type BaseExportFields struct {
	IndvId                    int64
	UserId                    int64
	SessionId                 int64
	PageId                    int64
	UserCreated               time.Time
	UserAppKey                string
	UserDisplayName           string
	UserEmail                 string
	EventStart                time.Time
	EventType                 string
	EventSubType              string
	EventCustomName           string
	EventTargetText           string
	EventTargetSelector       string
	EventPageOffset           int64
	EventSessionOffset        int64
	EventModFrustrated        int64
	EventModDead              int64
	EventModError             int64
	EventModSuspicious        int64
	EventWebSourceFileUrl     string
	SessionStart              time.Time
	PageStart                 time.Time
	PageDuration              int64
	PageActiveDuration        int64
	PageUrl                   string
	PageRefererUrl            string
	PageIp                    string
	PageLatLong               string
	PageUserAgent             string
	PageBrowser               string
	PageBrowserVersion        string
	PageDevice                string
	PagePlatform              string
	PageOperatingSystem       string
	PageScreenWidth           int64
	PageScreenHeight          int64
	PageViewportWidth         int64
	PageViewportHeight        int64
	PageNumInfos              int64
	PageNumWarnings           int64
	PageNumErrors             int64
	PageClusterId             int64
	PageMaxScrollDepthPercent int64
	LoadDomContentTime        int64
	LoadEventTime             int64
	LoadFirstPaintTime        int64
	ReqUrl                    string
	ReqMethod                 string
	ReqStatus                 int64
	CustomVars                string
}

type BigQuery 

type BigQuery struct {
	// contains filtered or unexported fields
}

func NewBigQuery 

func NewBigQuery(c *config.BigQueryConfig) *BigQuery

func (*BigQuery) ApplyExportSchema 

func (bq *BigQuery) ApplyExportSchema(s Schema) error

func (*BigQuery) GetExportTableColumns 

func (bq *BigQuery) GetExportTableColumns() []string

GetExportTableColumns returns a slice of the columns in the existing export table

func (*BigQuery) GetMissingFields 

func (bq *BigQuery) GetMissingFields(hauserSchema, bqSchema bigquery.Schema) []*bigquery.FieldSchema

GetMissingFields returns all fields that are present in the hauserSchema, but not in the bqSchema

func (*BigQuery) InitExportTable 

func (bq *BigQuery) InitExportTable(s Schema) (bool, error)

func (*BigQuery) LastSyncPoint 

func (bq *BigQuery) LastSyncPoint(_ context.Context) (time.Time, error)

func (*BigQuery) LoadToWarehouse 

func (bq *BigQuery) LoadToWarehouse(storageRef string, startTime time.Time) error

func (*BigQuery) SaveSyncPoint 

func (bq *BigQuery) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*BigQuery) ValueToString 

func (bq *BigQuery) ValueToString(val interface{}, isTime bool) string

type Database 

type Database interface {
	Syncable
	LoadToWarehouse(storageRef string, start time.Time) error
	ValueToString(val interface{}, isTime bool) string
	GetExportTableColumns() []string

	// InitExportTable should attempt to create the table in the database. If the table doesn't exist, the provided
	// schema should be applied to the table and this function should return `true`, assuming an error didn't occur.
	// If the table existed, this should return false to signal that follow-up schema validation is necessary.
	InitExportTable(Schema) (bool, error)

	// ApplyExportSchema will attempt to update the schema in the database to the provided schema.
	// The provided schema must be compatible, or this will fail. Compatible schemas will have existing columns
	// in the same order as they are currently ordered in the table and can also add new columns to the end.
	ApplyExportSchema(Schema) error
}

type GCSStorage 

type GCSStorage struct {
	// contains filtered or unexported fields
}

func NewGCSStorage 

func NewGCSStorage(conf *config.GCSConfig, gcsClient *storage.Client) *GCSStorage

func (*GCSStorage) DeleteFile 

func (g *GCSStorage) DeleteFile(ctx context.Context, name string) error

func (*GCSStorage) GetFilePrefix added in v1.1.0 

func (g *GCSStorage) GetFilePrefix() string

func (*GCSStorage) GetFileReference 

func (g *GCSStorage) GetFileReference(name string) string

func (*GCSStorage) LastSyncPoint 

func (g *GCSStorage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*GCSStorage) ReadFile 

func (g *GCSStorage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*GCSStorage) SaveFile 

func (g *GCSStorage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*GCSStorage) SaveSyncPoint 

func (g *GCSStorage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type LocalDisk 

type LocalDisk struct {
	// contains filtered or unexported fields
}

func NewLocalDisk 

func NewLocalDisk(c *config.LocalConfig) *LocalDisk

func (*LocalDisk) DeleteFile 

func (w *LocalDisk) DeleteFile(_ context.Context, name string) error

DeleteFile should do nothing for local disk

func (*LocalDisk) GetFilePrefix added in v1.1.0 

func (w *LocalDisk) GetFilePrefix() string

func (*LocalDisk) GetFileReference 

func (w *LocalDisk) GetFileReference(name string) string

func (*LocalDisk) LastSyncPoint 

func (w *LocalDisk) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*LocalDisk) ReadFile 

func (w *LocalDisk) ReadFile(_ context.Context, name string) (io.Reader, error)

func (*LocalDisk) SaveFile 

func (w *LocalDisk) SaveFile(_ context.Context, name string, reader io.Reader) (string, error)

func (*LocalDisk) SaveSyncPoint 

func (w *LocalDisk) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type MobileFields 

type MobileFields struct {
	AppName               string
	AppPackageName        string
	AppDeviceModel        string
	AppDeviceVendor       string
	AppVersion            string
	AppOsVersion          string
	AppViewName           string
	EventMobileSourceFile string
}

Mobile Apps fields will not be available for accounts that do not have the feature.

type Redshift 

type Redshift struct {
	// contains filtered or unexported fields
}

func NewRedshift 

func NewRedshift(c *config.RedshiftConfig) *Redshift

func (*Redshift) ApplyExportSchema 

func (rs *Redshift) ApplyExportSchema(newSchema Schema) error

func (*Redshift) CopyInData 

func (rs *Redshift) CopyInData(s3file string) error

CopyInData copies data from the given s3File to the export table

func (*Redshift) CreateSyncTable 

func (rs *Redshift) CreateSyncTable() error

CreateSyncTable creates a sync table with the hauser sync table schema

func (*Redshift) DeleteExportRecordsAfter 

func (rs *Redshift) DeleteExportRecordsAfter(end time.Time) error

func (*Redshift) DoesTableExist 

func (rs *Redshift) DoesTableExist(name string) bool

DoesTableExist checks if a table with a given name exists

func (*Redshift) GetExportTableColumns 

func (rs *Redshift) GetExportTableColumns() []string

GetExportTableColumns returns all the columns of the export table. It opens a connection and calls getTableColumns

func (*Redshift) InitExportTable 

func (rs *Redshift) InitExportTable(schema Schema) (bool, error)

func (*Redshift) LastSyncPoint 

func (rs *Redshift) LastSyncPoint(_ context.Context) (time.Time, error)

func (*Redshift) LoadToWarehouse 

func (rs *Redshift) LoadToWarehouse(s3obj string, _ time.Time) error

func (*Redshift) MakeRedshiftConnection 

func (rs *Redshift) MakeRedshiftConnection() (*sql.DB, error)

func (*Redshift) RemoveOrphanedRecords 

func (rs *Redshift) RemoveOrphanedRecords(lastSync pq.NullTime) error

func (*Redshift) SaveSyncPoint 

func (rs *Redshift) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*Redshift) ValueToString 

func (rs *Redshift) ValueToString(val interface{}, isTime bool) string

type S3Storage 

type S3Storage struct {
	// contains filtered or unexported fields
}

func NewS3Storage 

func NewS3Storage(conf *config.S3Config) *S3Storage

func (*S3Storage) DeleteFile 

func (s *S3Storage) DeleteFile(ctx context.Context, name string) error

func (*S3Storage) GetFilePrefix added in v1.1.0 

func (s *S3Storage) GetFilePrefix() string

func (*S3Storage) GetFileReference 

func (s *S3Storage) GetFileReference(name string) string

func (*S3Storage) LastSyncPoint 

func (s *S3Storage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*S3Storage) ReadFile 

func (s *S3Storage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*S3Storage) SaveFile 

func (s *S3Storage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*S3Storage) SaveSyncPoint 

func (s *S3Storage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Schema 

type Schema []WarehouseField

func MakeSchema 

func MakeSchema(vals ...interface{}) Schema

func (Schema) Equals 

func (s Schema) Equals(other Schema) bool

func (Schema) GetFieldForName 

func (s Schema) GetFieldForName(col string) WarehouseField

GetFieldForName takes an existing column name and returns the matching schema field. It performs some conversions between the legacy field names and the new field names.

func (Schema) GetFullStoryFields 

func (s Schema) GetFullStoryFields() []string

func (Schema) GetMissingFieldsFor 

func (s Schema) GetMissingFieldsFor(b Schema) []WarehouseField

func (Schema) IsCompatibleWith 

func (s Schema) IsCompatibleWith(other Schema) bool

func (Schema) ReconcileWithExisting 

func (s Schema) ReconcileWithExisting(colNames []string) Schema

ReconcileWithExisting returns a new schema that is compatible with the provided column names. If exported fields are missing from the column list, they are appended to the end.

type Storage 

type Storage interface {
	Syncable
	SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)
	ReadFile(ctx context.Context, name string) (io.Reader, error)
	DeleteFile(ctx context.Context, name string) error
	GetFileReference(name string) string
	GetFilePrefix() string
}

type SyncViaStorageMixin 

type SyncViaStorageMixin struct {
	// contains filtered or unexported fields
}

SyncViaStorageMixin provides a default implementation for the Syncable interface.

func (SyncViaStorageMixin) LastSyncPoint 

func (s SyncViaStorageMixin) LastSyncPoint(ctx context.Context) (time.Time, error)

func (SyncViaStorageMixin) SaveSyncPoint 

func (s SyncViaStorageMixin) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Syncable 

type Syncable interface {
	LastSyncPoint(ctx context.Context) (time.Time, error)
	SaveSyncPoint(ctx context.Context, endTime time.Time) error
}

type ValueToStringFn 

type ValueToStringFn func(val interface{}, isTime bool) string

type WarehouseField 

type WarehouseField struct {
	// The name of the field as it exists in the database.
	// By default this will match the `FullStoryFieldName`, but may not match for certain fields
	// if they've been renamed in the export. (e.g. "PageAgent").
	// If the database contains columns that are not part of the FullStory export, this field will still be populated.
	DBName string

	// The name of the field from FullStory.
	// This can be empty if an existing database table contains columns that FullStory does not include
	// in the export. This value, if non-empty, is used as part of the `CreateExport` request in the
	// `DataExportClient`
	FullStoryFieldName string

	// FieldType should be used by each database implementation to specify the datatype
	// for this column. This is only used when creating or modifying a database's schema.
	// If the `FullStoryFieldName` is blank, then this should be nil.
	FieldType reflect.Type
}

WarehouseField contains metadata for a field/column in the warehouse.

func (WarehouseField) IsTime 

func (f WarehouseField) IsTime() bool

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.