warehouse

package
v1.5.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 5, 2024 License: MIT Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source
const RFC3339Micro = "2006-01-02T15:04:05.999999Z07:00"

Variables

View Source
var (
	ErrFileNotFound = errors.New("file not found")
)

Functions

func IndexField

func IndexField(needle WarehouseField, haystack Schema) int

func ValueToString

func ValueToString(val interface{}, isTime bool) string

ValueToString is a common interface method that implementations use to perform value to string conversion

Types

type BaseExportFields

type BaseExportFields struct {
	IndvId                     int64
	UserId                     int64
	SessionId                  int64
	PageId                     int64
	UserCreated                time.Time
	UserAppKey                 string
	UserDisplayName            string
	UserEmail                  string
	EventStart                 time.Time
	EventType                  string
	EventSubType               string
	EventCustomName            string
	EventTargetText            string
	EventTargetSelector        string
	EventPageOffset            int64
	EventSessionOffset         int64
	EventModFrustrated         int64
	EventModDead               int64
	EventModError              int64
	EventModSuspicious         int64
	EventVarErrorKind          string
	EventVarFields             string
	EventWebSourceFileUrl      string
	EventFirstInputDelay       int64
	EventCumulativeLayoutShift float64
	SessionStart               time.Time
	PageName                   string
	PageStart                  time.Time
	PageDuration               int64
	PageActiveDuration         int64
	PageUrl                    string
	PageRefererUrl             string
	PageIp                     string
	PageLatLong                string
	PageUserAgent              string
	PageBrowser                string
	PageBrowserVersion         string
	PageDevice                 string
	PagePlatform               string
	PageOperatingSystem        string
	PageScreenWidth            int64
	PageScreenHeight           int64
	PageViewportWidth          int64
	PageViewportHeight         int64
	PageNumEvents              int32
	PageNumDerivedEvents       int32
	PageNumInfos               int64
	PageNumWarnings            int64
	PageNumErrors              int64
	PageClusterId              int64
	PageMaxScrollDepthPercent  int64
	LoadDomContentTime         int64
	LoadEventTime              int64
	LoadFirstPaintTime         int64
	LoadLargestPaintTime       int64
	ReqUrl                     string
	ReqMethod                  string
	ReqStatus                  int64
	CustomVars                 string
}

type BigQuery

type BigQuery struct {
	// contains filtered or unexported fields
}

func NewBigQuery

func NewBigQuery(c *config.BigQueryConfig) *BigQuery

func (*BigQuery) ApplyExportSchema

func (bq *BigQuery) ApplyExportSchema(s Schema) error

func (*BigQuery) GetExportTableColumns

func (bq *BigQuery) GetExportTableColumns() []string

GetExportTableColumns returns a slice of the columns in the existing export table

func (*BigQuery) GetMissingFields

func (bq *BigQuery) GetMissingFields(hauserSchema, bqSchema bigquery.Schema) []*bigquery.FieldSchema

GetMissingFields returns all fields that are present in the hauserSchema, but not in the bqSchema

func (*BigQuery) InitExportTable

func (bq *BigQuery) InitExportTable(s Schema) (bool, error)

func (*BigQuery) LastSyncPoint

func (bq *BigQuery) LastSyncPoint(_ context.Context) (time.Time, error)

func (*BigQuery) LoadToWarehouse

func (bq *BigQuery) LoadToWarehouse(storageRef string, startTime time.Time) error

func (*BigQuery) SaveSyncPoint

func (bq *BigQuery) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*BigQuery) ValueToString

func (bq *BigQuery) ValueToString(val interface{}, isTime bool) string

type Database

type Database interface {
	Syncable
	LoadToWarehouse(storageRef string, start time.Time) error
	ValueToString(val interface{}, isTime bool) string
	GetExportTableColumns() []string

	// InitExportTable should attempt to create the table in the database. If the table doesn't exist, the provided
	// schema should be applied to the table and this function should return `true`, assuming an error didn't occur.
	// If the table existed, this should return false to signal that follow-up schema validation is necessary.
	InitExportTable(Schema) (bool, error)

	// ApplyExportSchema will attempt to update the schema in the database to the provided schema.
	// The provided schema must be compatible, or this will fail. Compatible schemas will have existing columns
	// in the same order as they are currently ordered in the table and can also add new columns to the end.
	ApplyExportSchema(Schema) error
}

type GCSStorage

type GCSStorage struct {
	// contains filtered or unexported fields
}

func NewGCSStorage

func NewGCSStorage(conf *config.GCSConfig, gcsClient *storage.Client) *GCSStorage

func (*GCSStorage) DeleteFile

func (g *GCSStorage) DeleteFile(ctx context.Context, name string) error

func (*GCSStorage) GetFilePrefix added in v1.1.0

func (g *GCSStorage) GetFilePrefix() string

func (*GCSStorage) GetFileReference

func (g *GCSStorage) GetFileReference(name string) string

func (*GCSStorage) LastSyncPoint

func (g *GCSStorage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*GCSStorage) ReadFile

func (g *GCSStorage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*GCSStorage) SaveFile

func (g *GCSStorage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*GCSStorage) SaveSyncPoint

func (g *GCSStorage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type LocalDisk

type LocalDisk struct {
	// contains filtered or unexported fields
}

func NewLocalDisk

func NewLocalDisk(c *config.LocalConfig) *LocalDisk

func (*LocalDisk) DeleteFile

func (w *LocalDisk) DeleteFile(_ context.Context, name string) error

DeleteFile should do nothing for local disk

func (*LocalDisk) GetFilePrefix added in v1.1.0

func (w *LocalDisk) GetFilePrefix() string

func (*LocalDisk) GetFileReference

func (w *LocalDisk) GetFileReference(name string) string

func (*LocalDisk) LastSyncPoint

func (w *LocalDisk) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*LocalDisk) ReadFile

func (w *LocalDisk) ReadFile(_ context.Context, name string) (io.Reader, error)

func (*LocalDisk) SaveFile

func (w *LocalDisk) SaveFile(_ context.Context, name string, reader io.Reader) (string, error)

func (*LocalDisk) SaveSyncPoint

func (w *LocalDisk) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type MobileFields

type MobileFields struct {
	AppName               string
	AppPackageName        string
	AppDeviceModel        string
	AppDeviceVendor       string
	AppVersion            string
	AppOsVersion          string
	AppViewName           string
	EventMobileSourceFile string
}

Mobile Apps fields will not be available for accounts that do not have the feature.

type Redshift

type Redshift struct {
	// contains filtered or unexported fields
}

func NewRedshift

func NewRedshift(c *config.RedshiftConfig) *Redshift

func (*Redshift) ApplyExportSchema

func (rs *Redshift) ApplyExportSchema(newSchema Schema) error

func (*Redshift) CopyInData

func (rs *Redshift) CopyInData(s3file string) error

CopyInData copies data from the given s3File to the export table

func (*Redshift) CreateSyncTable

func (rs *Redshift) CreateSyncTable() error

CreateSyncTable creates a sync table with the hauser sync table schema

func (*Redshift) DeleteExportRecordsAfter

func (rs *Redshift) DeleteExportRecordsAfter(end time.Time) error

func (*Redshift) DoesTableExist

func (rs *Redshift) DoesTableExist(name string) bool

DoesTableExist checks if a table with a given name exists

func (*Redshift) GetExportTableColumns

func (rs *Redshift) GetExportTableColumns() []string

GetExportTableColumns returns all the columns of the export table. It opens a connection and calls getTableColumns

func (*Redshift) InitExportTable

func (rs *Redshift) InitExportTable(schema Schema) (bool, error)

func (*Redshift) LastSyncPoint

func (rs *Redshift) LastSyncPoint(_ context.Context) (time.Time, error)

func (*Redshift) LoadToWarehouse

func (rs *Redshift) LoadToWarehouse(s3obj string, _ time.Time) error

func (*Redshift) MakeRedshiftConnection

func (rs *Redshift) MakeRedshiftConnection() (*sql.DB, error)

func (*Redshift) RemoveOrphanedRecords

func (rs *Redshift) RemoveOrphanedRecords(lastSync pq.NullTime) error

func (*Redshift) SaveSyncPoint

func (rs *Redshift) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*Redshift) ValueToString

func (rs *Redshift) ValueToString(val interface{}, isTime bool) string

type S3Storage

type S3Storage struct {
	// contains filtered or unexported fields
}

func NewS3Storage

func NewS3Storage(conf *config.S3Config) *S3Storage

func (*S3Storage) DeleteFile

func (s *S3Storage) DeleteFile(ctx context.Context, name string) error

func (*S3Storage) GetFilePrefix added in v1.1.0

func (s *S3Storage) GetFilePrefix() string

func (*S3Storage) GetFileReference

func (s *S3Storage) GetFileReference(name string) string

func (*S3Storage) LastSyncPoint

func (s *S3Storage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*S3Storage) ReadFile

func (s *S3Storage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*S3Storage) SaveFile

func (s *S3Storage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*S3Storage) SaveSyncPoint

func (s *S3Storage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Schema

type Schema []WarehouseField

func MakeSchema

func MakeSchema(vals ...interface{}) Schema

func (Schema) Equals

func (s Schema) Equals(other Schema) bool

func (Schema) GetFieldForName

func (s Schema) GetFieldForName(col string) WarehouseField

GetFieldForName takes an existing column name and returns the matching schema field. It performs some conversions between the legacy field names and the new field names.

func (Schema) GetFullStoryFields

func (s Schema) GetFullStoryFields() []string

func (Schema) GetMissingFieldsFor

func (s Schema) GetMissingFieldsFor(b Schema) []WarehouseField

func (Schema) IsCompatibleWith

func (s Schema) IsCompatibleWith(other Schema) bool

func (Schema) ReconcileWithExisting

func (s Schema) ReconcileWithExisting(colNames []string) Schema

ReconcileWithExisting returns a new schema that is compatible with the provided column names. If exported fields are missing from the column list, they are appended to the end.

type Storage

type Storage interface {
	Syncable
	SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)
	ReadFile(ctx context.Context, name string) (io.Reader, error)
	DeleteFile(ctx context.Context, name string) error
	GetFileReference(name string) string
	GetFilePrefix() string
}

type SyncViaStorageMixin

type SyncViaStorageMixin struct {
	// contains filtered or unexported fields
}

SyncViaStorageMixin provides a default implementation for the Syncable interface.

func (SyncViaStorageMixin) LastSyncPoint

func (s SyncViaStorageMixin) LastSyncPoint(ctx context.Context) (time.Time, error)

func (SyncViaStorageMixin) SaveSyncPoint

func (s SyncViaStorageMixin) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Syncable

type Syncable interface {
	LastSyncPoint(ctx context.Context) (time.Time, error)
	SaveSyncPoint(ctx context.Context, endTime time.Time) error
}

type ValueToStringFn

type ValueToStringFn func(val interface{}, isTime bool) string

type WarehouseField

type WarehouseField struct {
	// The name of the field as it exists in the database.
	// By default this will match the `FullStoryFieldName`, but may not match for certain fields
	// if they've been renamed in the export. (e.g. "PageAgent").
	// If the database contains columns that are not part of the FullStory export, this field will still be populated.
	DBName string

	// The name of the field from FullStory.
	// This can be empty if an existing database table contains columns that FullStory does not include
	// in the export. This value, if non-empty, is used as part of the `CreateExport` request in the
	// `DataExportClient`
	FullStoryFieldName string

	// FieldType should be used by each database implementation to specify the datatype
	// for this column. This is only used when creating or modifying a database's schema.
	// If the `FullStoryFieldName` is blank, then this should be nil.
	FieldType reflect.Type
}

WarehouseField contains metadata for a field/column in the warehouse.

func (WarehouseField) IsTime

func (f WarehouseField) IsTime() bool

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL