generator

Documentation

Index

• func CurrentTimeMicros() int64
• func GenerateDoc(spec DocumentSpec) (interface{}, error)
• func GenerateDocs(spec DocumentSpec) ([]interface{}, error)
• func GeneratePatches(num_patch int, destination string, c chan map[string]interface{}) ([]interface{}, error)
• func RandomFieldAdd(destination string, c chan map[string]interface{})
• func RandomFieldReplace(destination string, c chan map[string]interface{})
• func RandomString(n int) string
• func RecordE2ELatency(latency float64)
• func SetMaxDoc(maxDocId int)
• type AddressStruct
• type CoordinatesStruct
• type Destination
• type DocStruct
• type DocumentSpec
• type Elastic
  • func (e *Elastic) ConfigureDestination() error
  • func (e *Elastic) GetLatestTimestamp() (time.Time, error)
  • func (e *Elastic) SendDocument(docs []any) error
  • func (e *Elastic) SendPatch(docs []interface{}) error
• type FriendDetailsStruct
• type FriendStruct
• type NameStruct
• type Null
  • func (n *Null) ConfigureDestination() error
  • func (n *Null) GetLatestTimestamp() (time.Time, error)
  • func (n *Null) SendDocument(docs []any) error
  • func (n *Null) SendPatch(docs []interface{}) error
• type Rockset
  • func (r *Rockset) ConfigureDestination() error
  • func (r *Rockset) GetLatestTimestamp() (time.Time, error)
  • func (r *Rockset) SendDocument(docs []any) error
  • func (r *Rockset) SendPatch(docs []interface{}) error
• type Snowflake
  • func (r *Snowflake) ConfigureDestination() error
  • func (r *Snowflake) GetLatestTimestamp() (time.Time, error)
  • func (r *Snowflake) SendDocument(docs []any) error
  • func (r *Snowflake) SendPatch(docs []interface{}) error

Functions

func CurrentTimeMicros

func CurrentTimeMicros() int64

func GenerateDoc

func GenerateDoc(spec DocumentSpec) (interface{}, error)

func GenerateDocs

func GenerateDocs(spec DocumentSpec) ([]interface{}, error)

func GeneratePatches

func GeneratePatches(num_patch int, destination string, c chan map[string]interface{}) ([]interface{}, error)

func RandomFieldAdd

func RandomFieldAdd(destination string, c chan map[string]interface{})

func RandomFieldReplace

func RandomFieldReplace(destination string, c chan map[string]interface{})

func RandomString

func RandomString(n int) string

func RecordE2ELatency

func RecordE2ELatency(latency float64)

func SetMaxDoc

func SetMaxDoc(maxDocId int)

Types

type AddressStruct

type AddressStruct struct {
	Street      string `faker:"oneof: 1st, 2nd, 3rd, 4th, 5th, 6th, 7th, 8th, 9th, 10th"`
	City        string `faker:"oneof: SF, San Mateo, San Jose, Mountain View, Menlo Park, Palo Alto"`
	ZipCode     int16
	Coordinates CoordinatesStruct
}

type CoordinatesStruct

type CoordinatesStruct struct {
	Latitude  float32 `faker:"lat"`
	Longitude float32 `faker:"long"`
}

type Destination

type Destination interface {
	// SendDocument sends a batch of documents to the destination.
	SendDocument(docs []any) error

	// Send a batch of patches to the destination.
	SendPatch(docs []any) error

	// GetLatestTimestamp get latest timestamp seen in the destination.
	GetLatestTimestamp() (time.Time, error)

	// ConfigureDestination is used to make any configuration changes to the destination that might be required for sending documents.
	ConfigureDestination() error
}

Destination is where to send the generated documents to

type DocStruct

type DocStruct struct {
	Guid       string
	IsActive   bool
	Balance    float64 `faker:"amount"`
	Picture    string
	Age        int `faker:"oneof: 15, 27, 61"`
	Name       NameStruct
	Company    string `faker:"oneof: facebook, google, rockset, tesla, uber, lyft"`
	Email      string `faker:"email"`
	Phone      string `faker:"phone_number"`
	Address    AddressStruct
	About      string   `faker:"sentence"`
	Registered string   `faker:"timestamp"`
	Tags       []string `faker:"slice_len=9,len=14"`
	Friends    FriendStruct
	Greeting   string `faker:"paragraph"`
}

type DocumentSpec

type DocumentSpec struct {
	Destination          string
	GeneratorIdentifier  string
	BatchSize            int
	Mode                 string
	IdMode               string
	UpdatePercentage     int
	NumClusters          int
	HotClusterPercentage int
}

type Elastic

type Elastic struct {
	Auth                string
	URL                 string
	IndexName           string
	Client              *http.Client
	GeneratorIdentifier string
}

Elastic contains all configurations needed to send documents to Elastic

func (*Elastic) ConfigureDestination

func (e *Elastic) ConfigureDestination() error

func (*Elastic) GetLatestTimestamp

func (e *Elastic) GetLatestTimestamp() (time.Time, error)

GetLatestTimestamp returns the latest _event_time in Rockset

func (*Elastic) SendDocument

func (e *Elastic) SendDocument(docs []any) error

SendDocument sends a batch of documents to Elastic

func (*Elastic) SendPatch

func (e *Elastic) SendPatch(docs []interface{}) error

type FriendDetailsStruct

type FriendDetailsStruct struct {
	Name NameStruct
	Age  int `faker:"oneof: 15, 27, 61"`
}

type FriendStruct

type FriendStruct struct {
	Friend1 FriendDetailsStruct
	Friend2 FriendDetailsStruct
	Friend3 FriendDetailsStruct
	Friend4 FriendDetailsStruct
	Friend5 FriendDetailsStruct
}

type NameStruct

type NameStruct struct {
	First string `faker:"first_name"`
	Last  string `faker:"last_name"`
}

type Null

type Null struct{}

Null destination for local testing

func (*Null) ConfigureDestination

func (n *Null) ConfigureDestination() error

func (*Null) GetLatestTimestamp

func (n *Null) GetLatestTimestamp() (time.Time, error)

func (*Null) SendDocument

func (n *Null) SendDocument(docs []any) error

func (*Null) SendPatch

func (n *Null) SendPatch(docs []interface{}) error

type Rockset

type Rockset struct {
	APIKey              string
	APIServer           string
	CollectionPath      string
	Client              *http.Client
	GeneratorIdentifier string
}

Rockset contains all configurations needed to send documents to Rockset

func (*Rockset) ConfigureDestination

func (r *Rockset) ConfigureDestination() error

func (*Rockset) GetLatestTimestamp

func (r *Rockset) GetLatestTimestamp() (time.Time, error)

GetLatestTimestamp returns the latest _event_time in Rockset

func (*Rockset) SendDocument

func (r *Rockset) SendDocument(docs []any) error

SendDocument sends a batch of documents to Rockset

func (*Rockset) SendPatch

func (r *Rockset) SendPatch(docs []interface{}) error

type Snowflake

type Snowflake struct {
	Account             string
	User                string
	Password            string
	Warehouse           string
	Database            string
	Schema              string
	GeneratorIdentifier string
	StageS3BucketName   string
	AWSRegion           string
	Table               string
	DBConnection        *sql.DB
}

Snowflake contains all configurations needed to send documents to Snowflake

func (*Snowflake) ConfigureDestination

func (r *Snowflake) ConfigureDestination() error

ConfigureDestination is used to make configuration changes to the Snowflake instance for sending documents.

func (*Snowflake) GetLatestTimestamp

func (r *Snowflake) GetLatestTimestamp() (time.Time, error)

GetLatestTimestamp returns the latest _event_time in Snowflake

func (*Snowflake) SendDocument

func (r *Snowflake) SendDocument(docs []any) error

SendDocument sends a batch of documents to Snowflake

func (*Snowflake) SendPatch

func (r *Snowflake) SendPatch(docs []interface{}) error