Documentation

Index

Constants

View Source
const RFC3339Micro = "2006-01-02T15:04:05.999999Z07:00"

Variables

View Source
var (
	ErrFileNotFound = errors.New("file not found")
)

Functions

func IndexField

func IndexField(needle WarehouseField, haystack Schema) int

func ValueToString

func ValueToString(val interface{}, isTime bool) string

    ValueToString is a common interface method that implementations use to perform value to string conversion

    Types

    type BaseExportFields

    type BaseExportFields struct {
    	IndvId              int64
    	UserId              int64
    	SessionId           int64
    	PageId              int64
    	UserCreated         time.Time
    	UserAppKey          string
    	UserDisplayName     string
    	UserEmail           string
    	EventStart          time.Time
    	EventType           string
    	EventCustomName     string
    	EventTargetText     string
    	EventTargetSelector string
    	EventModFrustrated  int64
    	EventModDead        int64
    	EventModError       int64
    	EventModSuspicious  int64
    	SessionStart        time.Time
    	PageStart           time.Time
    	PageDuration        int64
    	PageActiveDuration  int64
    	PageUrl             string
    	PageRefererUrl      string
    	PageIp              string
    	PageLatLong         string
    	PageUserAgent       string
    	PageBrowser         string
    	PageDevice          string
    	PagePlatform        string
    	PageOperatingSystem string
    	PageScreenWidth     int64
    	PageScreenHeight    int64
    	PageViewportWidth   int64
    	PageViewportHeight  int64
    	PageNumInfos        int64
    	PageNumWarnings     int64
    	PageNumErrors       int64
    	PageClusterId       int64
    	LoadDomContentTime  int64
    	LoadEventTime       int64
    	LoadFirstPaintTime  int64
    	CustomVars          string
    }

    type BigQuery

    type BigQuery struct {
    	// contains filtered or unexported fields
    }

    func NewBigQuery

    func NewBigQuery(c *config.BigQueryConfig) *BigQuery

    func (*BigQuery) ApplyExportSchema

    func (bq *BigQuery) ApplyExportSchema(s Schema) error

    func (*BigQuery) GetExportTableColumns

    func (bq *BigQuery) GetExportTableColumns() []string

      GetExportTableColumns returns a slice of the columns in the existing export table

      func (*BigQuery) GetMissingFields

      func (bq *BigQuery) GetMissingFields(hauserSchema, bqSchema bigquery.Schema) []*bigquery.FieldSchema

        GetMissingFields returns all fields that are present in the hauserSchema, but not in the bqSchema

        func (*BigQuery) InitExportTable

        func (bq *BigQuery) InitExportTable(s Schema) (bool, error)

        func (*BigQuery) LastSyncPoint

        func (bq *BigQuery) LastSyncPoint(_ context.Context) (time.Time, error)

        func (*BigQuery) LoadToWarehouse

        func (bq *BigQuery) LoadToWarehouse(storageRef string, startTime time.Time) error

        func (*BigQuery) SaveSyncPoint

        func (bq *BigQuery) SaveSyncPoint(_ context.Context, endTime time.Time) error

        func (*BigQuery) ValueToString

        func (bq *BigQuery) ValueToString(val interface{}, isTime bool) string

        type Database

        type Database interface {
        	Syncable
        	LoadToWarehouse(storageRef string, start time.Time) error
        	ValueToString(val interface{}, isTime bool) string
        	GetExportTableColumns() []string
        
        	// InitExportTable should attempt to create the table in the database. If the table doesn't exist, the provided
        	// schema should be applied to the table and this function should return `true`, assuming an error didn't occur.
        	// If the table existed, this should return false to signal that follow-up schema validation is necessary.
        	InitExportTable(Schema) (bool, error)
        
        	// ApplyExportSchema will attempt to update the schema in the database to the provided schema.
        	// The provided schema must be compatible, or this will fail. Compatible schemas will have existing columns
        	// in the same order as they are currently ordered in the table and can also add new columns to the end.
        	ApplyExportSchema(Schema) error
        }

        type GCSStorage

        type GCSStorage struct {
        	// contains filtered or unexported fields
        }

        func NewGCSStorage

        func NewGCSStorage(conf *config.GCSConfig, gcsClient *storage.Client) *GCSStorage

        func (*GCSStorage) DeleteFile

        func (g *GCSStorage) DeleteFile(ctx context.Context, name string) error

        func (*GCSStorage) GetFileReference

        func (g *GCSStorage) GetFileReference(name string) string

        func (*GCSStorage) LastSyncPoint

        func (g *GCSStorage) LastSyncPoint(ctx context.Context) (time.Time, error)

        func (*GCSStorage) ReadFile

        func (g *GCSStorage) ReadFile(ctx context.Context, name string) (io.Reader, error)

        func (*GCSStorage) SaveFile

        func (g *GCSStorage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

        func (*GCSStorage) SaveSyncPoint

        func (g *GCSStorage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

        type LocalDisk

        type LocalDisk struct {
        	// contains filtered or unexported fields
        }

        func NewLocalDisk

        func NewLocalDisk(c *config.LocalConfig) *LocalDisk

        func (*LocalDisk) DeleteFile

        func (w *LocalDisk) DeleteFile(_ context.Context, name string) error

          DeleteFile should do nothing for local disk

          func (*LocalDisk) GetFileReference

          func (w *LocalDisk) GetFileReference(name string) string

          func (*LocalDisk) LastSyncPoint

          func (w *LocalDisk) LastSyncPoint(ctx context.Context) (time.Time, error)

          func (*LocalDisk) ReadFile

          func (w *LocalDisk) ReadFile(_ context.Context, name string) (io.Reader, error)

          func (*LocalDisk) SaveFile

          func (w *LocalDisk) SaveFile(_ context.Context, name string, reader io.Reader) (string, error)

          func (*LocalDisk) SaveSyncPoint

          func (w *LocalDisk) SaveSyncPoint(ctx context.Context, endTime time.Time) error

          type MobileFields

          type MobileFields struct {
          	AppName        string
          	AppPackageName string
          }

            Mobile Apps fields will not be available for accounts that do not have the feature.

            type Redshift

            type Redshift struct {
            	// contains filtered or unexported fields
            }

            func NewRedshift

            func NewRedshift(c *config.RedshiftConfig) *Redshift

            func (*Redshift) ApplyExportSchema

            func (rs *Redshift) ApplyExportSchema(newSchema Schema) error

            func (*Redshift) CopyInData

            func (rs *Redshift) CopyInData(s3file string) error

              CopyInData copies data from the given s3File to the export table

              func (*Redshift) CreateSyncTable

              func (rs *Redshift) CreateSyncTable() error

                CreateSyncTable creates a sync table with the hauser sync table schema

                func (*Redshift) DeleteExportRecordsAfter

                func (rs *Redshift) DeleteExportRecordsAfter(end time.Time) error

                func (*Redshift) DoesTableExist

                func (rs *Redshift) DoesTableExist(name string) bool

                  DoesTableExist checks if a table with a given name exists

                  func (*Redshift) GetExportTableColumns

                  func (rs *Redshift) GetExportTableColumns() []string

                    GetExportTableColumns returns all the columns of the export table. It opens a connection and calls getTableColumns

                    func (*Redshift) InitExportTable

                    func (rs *Redshift) InitExportTable(schema Schema) (bool, error)

                    func (*Redshift) LastSyncPoint

                    func (rs *Redshift) LastSyncPoint(_ context.Context) (time.Time, error)

                    func (*Redshift) LoadToWarehouse

                    func (rs *Redshift) LoadToWarehouse(s3obj string, _ time.Time) error

                    func (*Redshift) MakeRedshiftConnection

                    func (rs *Redshift) MakeRedshiftConnection() (*sql.DB, error)

                    func (*Redshift) RemoveOrphanedRecords

                    func (rs *Redshift) RemoveOrphanedRecords(lastSync pq.NullTime) error

                    func (*Redshift) SaveSyncPoint

                    func (rs *Redshift) SaveSyncPoint(_ context.Context, endTime time.Time) error

                    func (*Redshift) ValueToString

                    func (rs *Redshift) ValueToString(val interface{}, isTime bool) string

                    type S3Storage

                    type S3Storage struct {
                    	// contains filtered or unexported fields
                    }

                    func NewS3Storage

                    func NewS3Storage(conf *config.S3Config) *S3Storage

                    func (*S3Storage) DeleteFile

                    func (s *S3Storage) DeleteFile(ctx context.Context, name string) error

                    func (*S3Storage) GetFileReference

                    func (s *S3Storage) GetFileReference(name string) string

                    func (*S3Storage) LastSyncPoint

                    func (s *S3Storage) LastSyncPoint(ctx context.Context) (time.Time, error)

                    func (*S3Storage) ReadFile

                    func (s *S3Storage) ReadFile(ctx context.Context, name string) (io.Reader, error)

                    func (*S3Storage) SaveFile

                    func (s *S3Storage) SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)

                    func (*S3Storage) SaveSyncPoint

                    func (s *S3Storage) SaveSyncPoint(ctx context.Context, endTime time.Time) error

                    type Schema

                    type Schema []WarehouseField

                    func MakeSchema

                    func MakeSchema(vals ...interface{}) Schema

                    func (Schema) Equals

                    func (s Schema) Equals(other Schema) bool

                    func (Schema) GetFieldForName

                    func (s Schema) GetFieldForName(col string) WarehouseField

                      GetFieldForName takes an existing column name and returns the matching schema field. It performs some conversions between the legacy field names and the new field names.

                      func (Schema) GetFullStoryFields

                      func (s Schema) GetFullStoryFields() []string

                      func (Schema) GetMissingFieldsFor

                      func (s Schema) GetMissingFieldsFor(b Schema) []WarehouseField

                      func (Schema) IsCompatibleWith

                      func (s Schema) IsCompatibleWith(other Schema) bool

                      func (Schema) ReconcileWithExisting

                      func (s Schema) ReconcileWithExisting(colNames []string) Schema

                        ReconcileWithExisting returns a new schema that is compatible with the provided column names. If exported fields are missing from the column list, they are appended to the end.

                        type Storage

                        type Storage interface {
                        	Syncable
                        	SaveFile(ctx context.Context, name string, reader io.Reader) (string, error)
                        	ReadFile(ctx context.Context, name string) (io.Reader, error)
                        	DeleteFile(ctx context.Context, name string) error
                        	GetFileReference(name string) string
                        }

                        type SyncViaStorageMixin

                        type SyncViaStorageMixin struct {
                        	// contains filtered or unexported fields
                        }

                          SyncViaStorageMixin provides a default implementation for the Syncable interface.

                          func (SyncViaStorageMixin) LastSyncPoint

                          func (s SyncViaStorageMixin) LastSyncPoint(ctx context.Context) (time.Time, error)

                          func (SyncViaStorageMixin) SaveSyncPoint

                          func (s SyncViaStorageMixin) SaveSyncPoint(ctx context.Context, endTime time.Time) error

                          type Syncable

                          type Syncable interface {
                          	LastSyncPoint(ctx context.Context) (time.Time, error)
                          	SaveSyncPoint(ctx context.Context, endTime time.Time) error
                          }

                          type ValueToStringFn

                          type ValueToStringFn func(val interface{}, isTime bool) string

                          type WarehouseField

                          type WarehouseField struct {
                          	// The name of the field as it exists in the database.
                          	// By default this will match the `FullStoryFieldName`, but may not match for certain fields
                          	// if they've been renamed in the export. (e.g. "PageAgent").
                          	// If the database contains columns that are not part of the FullStory export, this field will still be populated.
                          	DBName string
                          
                          	// The name of the field from FullStory.
                          	// This can be empty if an existing database table contains columns that FullStory does not include
                          	// in the export. This value, if non-empty, is used as part of the `CreateExport` request in the
                          	// `DataExportClient`
                          	FullStoryFieldName string
                          
                          	// FieldType should be used by each database implementation to specify the datatype
                          	// for this column. This is only used when creating or modifying a database's schema.
                          	// If the `FullStoryFieldName` is blank, then this should be nil.
                          	FieldType reflect.Type
                          }

                            WarehouseField contains metadata for a field/column in the warehouse.

                            func (WarehouseField) IsTime

                            func (f WarehouseField) IsTime() bool