warehouse

package
v1.5.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 20, 2024 License: MIT Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const RFC3339Micro = "2006-01-02T15:04:05.999999Z07:00"

Variables

View Source 
var (
	ErrFileNotFound = errors.New("file not found")
)

Functions

func IndexField 

func IndexField(needle WarehouseField, haystack Schema) int

func ValueToString 

func ValueToString(val interface{}, isTime bool) string

ValueToString is a common interface method that implementations use to perform value to string conversion

Types

type BaseExportFields 

type BaseExportFields struct {
	IndvId                     int64
	UserId                     int64
	SessionId                  int64
	PageId                     int64
	UserCreated                time.Time
	UserAppKey                 string
	UserDisplayName            string
	UserEmail                  string
	EventStart                 time.Time
	EventType                  string
	EventSubType               string
	EventCustomName            string
	EventTargetText            string
	EventTargetSelector        string
	EventPageOffset            int64
	EventSessionOffset         int64
	EventModFrustrated         int64
	EventModDead               int64
	EventModError              int64
	EventModSuspicious         int64
	EventVarErrorKind          string
	EventVarFields             string
	EventWebSourceFileUrl      string
	EventFirstInputDelay       int64
	EventCumulativeLayoutShift float64
	SessionStart               time.Time
	PageName                   string
	PageStart                  time.Time
	PageDuration               int64
	PageActiveDuration         int64
	PageUrl                    string
	PageRefererUrl             string
	PageIp                     string
	PageLatLong                string
	PageUserAgent              string
	PageBrowser                string
	PageBrowserVersion         string
	PageDevice                 string
	PagePlatform               string
	PageOperatingSystem        string
	PageScreenWidth            int64
	PageScreenHeight           int64
	PageViewportWidth          int64
	PageViewportHeight         int64
	PageNumEvents              int32
	PageNumDerivedEvents       int32
	PageNumInfos               int64
	PageNumWarnings            int64
	PageNumErrors              int64
	PageClusterId              int64
	PageMaxScrollDepthPercent  int64
	LoadDomContentTime         int64
	LoadEventTime              int64
	LoadFirstPaintTime         int64
	LoadLargestPaintTime       int64
	ReqUrl                     string
	ReqMethod                  string
	ReqStatus                  int64
	CustomVars                 string
}

type BigQuery 

type BigQuery struct {
	// contains filtered or unexported fields
}

func NewBigQuery 

func NewBigQuery(c *config.BigQueryConfig) *BigQuery

func (*BigQuery) ApplyExportSchema 

func (bq *BigQuery) ApplyExportSchema(s Schema) error

func (*BigQuery) GetExportTableColumns 

func (bq *BigQuery) GetExportTableColumns() []string

GetExportTableColumns returns a slice of the columns in the existing export table

func (*BigQuery) GetMissingFields 

func (bq *BigQuery) GetMissingFields(hauserSchema, bqSchema bigquery.Schema) []*bigquery.FieldSchema

GetMissingFields returns all fields that are present in the hauserSchema, but not in the bqSchema

func (*BigQuery) InitExportTable 

func (bq *BigQuery) InitExportTable(s Schema) (bool, error)

func (*BigQuery) LastSyncPoint 

func (bq *BigQuery) LastSyncPoint(_ context.Context) (time.Time, error)

func (*BigQuery) LoadToWarehouse 

func (bq *BigQuery) LoadToWarehouse(storageRef string, startTime time.Time) error

func (*BigQuery) SaveSyncPoint 

func (bq *BigQuery) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*BigQuery) ValueToString 

func (bq *BigQuery) ValueToString(val interface{}, isTime bool) string

type Database 

type Database interface {
	Syncable
	LoadToWarehouse(storageRef string, start time.Time) error
	ValueToString(val interface{}, isTime bool) string
	GetExportTableColumns() []string

	// InitExportTable should attempt to create the table in the database. If the table doesn't exist, the provided
	// schema should be applied to the table and this function should return `true`, assuming an error didn't occur.
	// If the table existed, this should return false to signal that follow-up schema validation is necessary.
	InitExportTable(Schema) (bool, error)

	// ApplyExportSchema will attempt to update the schema in the database to the provided schema.
	// The provided schema must be compatible, or this will fail. Compatible schemas will have existing columns
	// in the same order as they are currently ordered in the table and can also add new columns to the end.
	ApplyExportSchema(Schema) error
}

type GCSStorage 

type GCSStorage struct {
	// contains filtered or unexported fields
}

func NewGCSStorage 

func NewGCSStorage(conf *config.GCSConfig, gcsClient *storage.Client) *GCSStorage

func (*GCSStorage) DeleteFile 

func (g *GCSStorage) DeleteFile(ctx context.Context, name string) error

func (*GCSStorage) GetFilePrefix added in v1.1.0 

func (g *GCSStorage) GetFilePrefix() string

func (*GCSStorage) GetFileReference 

func (g *GCSStorage) GetFileReference(name string) string

func (*GCSStorage) LastSyncPoint 

func (g *GCSStorage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*GCSStorage) ReadFile 

func (g *GCSStorage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*GCSStorage) SaveFile 

func (g *GCSStorage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*GCSStorage) SaveSyncPoint 

func (g *GCSStorage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type LocalDisk 

type LocalDisk struct {
	// contains filtered or unexported fields
}

func NewLocalDisk 

func NewLocalDisk(c *config.LocalConfig) *LocalDisk

func (*LocalDisk) DeleteFile 

func (w *LocalDisk) DeleteFile(_ context.Context, name string) error

DeleteFile should do nothing for local disk

func (*LocalDisk) GetFilePrefix added in v1.1.0 

func (w *LocalDisk) GetFilePrefix() string

func (*LocalDisk) GetFileReference 

func (w *LocalDisk) GetFileReference(name string) string

func (*LocalDisk) LastSyncPoint 

func (w *LocalDisk) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*LocalDisk) ReadFile 

func (w *LocalDisk) ReadFile(_ context.Context, name string) (io.Reader, error)

func (*LocalDisk) SaveFile 

func (w *LocalDisk) SaveFile(_ context.Context, name string, reader io.Reader) (string, error)

func (*LocalDisk) SaveSyncPoint 

func (w *LocalDisk) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type MobileFields 

type MobileFields struct {
	AppName               string
	AppPackageName        string
	AppDeviceModel        string
	AppDeviceVendor       string
	AppVersion            string
	AppOsVersion          string
	AppViewName           string
	EventMobileSourceFile string
}

Mobile Apps fields will not be available for accounts that do not have the feature.

type Redshift 

type Redshift struct {
	// contains filtered or unexported fields
}

func NewRedshift 

func NewRedshift(c *config.RedshiftConfig) *Redshift

func (*Redshift) ApplyExportSchema 

func (rs *Redshift) ApplyExportSchema(newSchema Schema) error

func (*Redshift) CopyInData 

func (rs *Redshift) CopyInData(s3file string) error

CopyInData copies data from the given s3File to the export table

func (*Redshift) CreateSyncTable 

func (rs *Redshift) CreateSyncTable() error

CreateSyncTable creates a sync table with the hauser sync table schema

func (*Redshift) DeleteExportRecordsAfter 

func (rs *Redshift) DeleteExportRecordsAfter(end time.Time) error

func (*Redshift) DoesTableExist 

func (rs *Redshift) DoesTableExist(name string) bool

DoesTableExist checks if a table with a given name exists

func (*Redshift) GetExportTableColumns 

func (rs *Redshift) GetExportTableColumns() []string

GetExportTableColumns returns all the columns of the export table. It opens a connection and calls getTableColumns

func (*Redshift) InitExportTable 

func (rs *Redshift) InitExportTable(schema Schema) (bool, error)

func (*Redshift) LastSyncPoint 

func (rs *Redshift) LastSyncPoint(_ context.Context) (time.Time, error)

func (*Redshift) LoadToWarehouse 

func (rs *Redshift) LoadToWarehouse(s3obj string, _ time.Time) error

func (*Redshift) MakeRedshiftConnection 

func (rs *Redshift) MakeRedshiftConnection() (*sql.DB, error)

func (*Redshift) RemoveOrphanedRecords 

func (rs *Redshift) RemoveOrphanedRecords(lastSync pq.NullTime) error

func (*Redshift) SaveSyncPoint 

func (rs *Redshift) SaveSyncPoint(_ context.Context, endTime time.Time) error

func (*Redshift) ValueToString 

func (rs *Redshift) ValueToString(val interface{}, isTime bool) string

type S3Storage 

type S3Storage struct {
	// contains filtered or unexported fields
}

func NewS3Storage 

func NewS3Storage(conf *config.S3Config) *S3Storage

func (*S3Storage) DeleteFile 

func (s *S3Storage) DeleteFile(ctx context.Context, name string) error

func (*S3Storage) GetFilePrefix added in v1.1.0 

func (s *S3Storage) GetFilePrefix() string

func (*S3Storage) GetFileReference 

func (s *S3Storage) GetFileReference(name string) string

func (*S3Storage) LastSyncPoint 

func (s *S3Storage) LastSyncPoint(ctx context.Context) (time.Time, error)

func (*S3Storage) ReadFile 

func (s *S3Storage) ReadFile(ctx context.Context, name string) (io.Reader, error)

func (*S3Storage) SaveFile 

func (s *S3Storage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

func (*S3Storage) SaveSyncPoint 

func (s *S3Storage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Schema 

type Schema []WarehouseField

func MakeSchema 

func MakeSchema(vals ...interface{}) Schema

func (Schema) Equals 

func (s Schema) Equals(other Schema) bool

func (Schema) GetFieldForName 

func (s Schema) GetFieldForName(col string) WarehouseField

GetFieldForName takes an existing column name and returns the matching schema field. It performs some conversions between the legacy field names and the new field names.

func (Schema) GetFullStoryFields 

func (s Schema) GetFullStoryFields() []string

func (Schema) GetMissingFieldsFor 

func (s Schema) GetMissingFieldsFor(b Schema) []WarehouseField

func (Schema) IsCompatibleWith 

func (s Schema) IsCompatibleWith(other Schema) bool

func (Schema) ReconcileWithExisting 

func (s Schema) ReconcileWithExisting(colNames []string) Schema

ReconcileWithExisting returns a new schema that is compatible with the provided column names. If exported fields are missing from the column list, they are appended to the end.

type Storage 

type Storage interface {
	Syncable
	SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)
	ReadFile(ctx context.Context, name string) (io.Reader, error)
	DeleteFile(ctx context.Context, name string) error
	GetFileReference(name string) string
	GetFilePrefix() string
}

type SyncViaStorageMixin 

type SyncViaStorageMixin struct {
	// contains filtered or unexported fields
}

SyncViaStorageMixin provides a default implementation for the Syncable interface.

func (SyncViaStorageMixin) LastSyncPoint 

func (s SyncViaStorageMixin) LastSyncPoint(ctx context.Context) (time.Time, error)

func (SyncViaStorageMixin) SaveSyncPoint 

func (s SyncViaStorageMixin) SaveSyncPoint(ctx context.Context, endTime time.Time) error

type Syncable 

type Syncable interface {
	LastSyncPoint(ctx context.Context) (time.Time, error)
	SaveSyncPoint(ctx context.Context, endTime time.Time) error
}

type ValueToStringFn 

type ValueToStringFn func(val interface{}, isTime bool) string

type WarehouseField 

type WarehouseField struct {
	// The name of the field as it exists in the database.
	// By default this will match the `FullStoryFieldName`, but may not match for certain fields
	// if they've been renamed in the export. (e.g. "PageAgent").
	// If the database contains columns that are not part of the FullStory export, this field will still be populated.
	DBName string

	// The name of the field from FullStory.
	// This can be empty if an existing database table contains columns that FullStory does not include
	// in the export. This value, if non-empty, is used as part of the `CreateExport` request in the
	// `DataExportClient`
	FullStoryFieldName string

	// FieldType should be used by each database implementation to specify the datatype
	// for this column. This is only used when creating or modifying a database's schema.
	// If the `FullStoryFieldName` is blank, then this should be nil.
	FieldType reflect.Type
}

WarehouseField contains metadata for a field/column in the warehouse.

func (WarehouseField) IsTime 

func (f WarehouseField) IsTime() bool

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.