Documentation ¶
Index ¶
- Variables
- func NoopFilter(map[string]string) bool
- type Action
- type AnnotatedKinesisConfig
- type AuthScoopSigner
- type BatcherConfig
- type ColumnDefinition
- type Config
- type EventFilterFunc
- type EventMetadataConfig
- type EventMetadataRow
- type EventMetadataType
- type FakeScoopSigner
- type FilterOperator
- type GlobberConfig
- type KinesisEventFilterConfig
- type KinesisWriterConfig
- type KinesisWriterEventConfig
- type LoadCheckRequest
- type LoadCheckResponse
- type LoadStatus
- type ManifestRowCopyRequest
- type Operation
- func NewAddOperation(outbound, inbound, type_, options, columns string) Operation
- func NewCancelDropEventOperation(reason string) Operation
- func NewDeleteOperation(outbound string) Operation
- func NewDropEventOperation(reason string) Operation
- func NewRenameOperation(current, new string) Operation
- func NewRequestDropEventOperation(reason string) Operation
- type RowCopyRequest
- type ScoopHealthCheck
- type ScoopSigner
- type TestableKinesisEventFilter
Constants ¶
This section is empty.
Variables ¶
var (
BadVerified error = errors.New("Bad Signature")
)
Functions ¶
func NoopFilter ¶
NoopFilter is a fiter func that always passes.
Types ¶
type Action ¶
type Action string
const ( ADD Action = "add" DELETE Action = "delete" RENAME Action = "rename" REQUEST_DROP_EVENT Action = "request_drop_event" // mark a table for manual deletion DROP_EVENT Action = "drop_event" // table has been dropped CANCEL_DROP_EVENT Action = "cancel_drop_event" // used to unmark the table for deletion )
type AnnotatedKinesisConfig ¶
type AnnotatedKinesisConfig struct { ID int AWSAccount int64 Team string Version int Contact string Usage string ConsumingLibrary string SpadeConfig KinesisWriterConfig LastEditedAt time.Time LastChangedBy string Dropped bool DroppedReason string }
AnnotatedKinesisConfig is a Kinesis configuration annotated with meta information.
type AuthScoopSigner ¶
type AuthScoopSigner struct { TimeSigner *msg_signer.TimeSigner Exp time.Duration }
func (*AuthScoopSigner) GetConfig ¶
func (s *AuthScoopSigner) GetConfig(body io.Reader) (*Config, error)
func (*AuthScoopSigner) GetRowCopyRequest ¶
func (s *AuthScoopSigner) GetRowCopyRequest(body io.Reader) (*RowCopyRequest, error)
func (*AuthScoopSigner) SignJsonBody ¶
func (s *AuthScoopSigner) SignJsonBody(o interface{}) ([]byte, error)
type BatcherConfig ¶
type BatcherConfig struct { // MaxSize is the max combined size of the batch MaxSize int // MaxEntries is the max number of entries that can be batched together // if batches does not have an entry limit, set MaxEntries as -1 MaxEntries int // MaxAge is the max age of the oldest entry in the glob MaxAge string // BufferLength is the length of the channel where newly // submitted entries are stored, decreasing the size of this // buffer can cause stalls, and increasing the size can increase // shutdown time BufferLength int }
BatcherConfig is used to configure a batcher instance
func (*BatcherConfig) Validate ¶
func (c *BatcherConfig) Validate() error
Validate returns an error if the batcher config is invalid, nil otherwise.
type ColumnDefinition ¶
type Config ¶
type Config struct { EventName string Columns []ColumnDefinition Version int }
type EventFilterFunc ¶
EventFilterFunc takes event properties and returns True if their values match desired conditions.
type EventMetadataConfig ¶
type EventMetadataConfig struct { // The first key is the event name, second key is metadata type Metadata map[string](map[string]EventMetadataRow) }
type EventMetadataRow ¶
type EventMetadataType ¶
type EventMetadataType string
const ( COMMENT EventMetadataType = "comment" EDGE_TYPE EventMetadataType = "edge_type" DATASTORES EventMetadataType = "datastores" BIRTH EventMetadataType = "birth" )
type FakeScoopSigner ¶
type FakeScoopSigner struct{}
func (*FakeScoopSigner) GetConfig ¶
func (s *FakeScoopSigner) GetConfig(b io.Reader) (*Config, error)
func (*FakeScoopSigner) GetRowCopyRequest ¶
func (s *FakeScoopSigner) GetRowCopyRequest(b io.Reader) (*RowCopyRequest, error)
func (*FakeScoopSigner) SignBody ¶
func (s *FakeScoopSigner) SignBody(b []byte) ([]byte, error)
does nothing
func (*FakeScoopSigner) SignJsonBody ¶
func (s *FakeScoopSigner) SignJsonBody(o interface{}) ([]byte, error)
type FilterOperator ¶
type FilterOperator string
FilterOperator represents the types of filter operations supported by KinesisEventFilterConfig.
const ( IN_SET FilterOperator = "in_set" NOT_IN_SET FilterOperator = "not_in_set" )
type GlobberConfig ¶
type GlobberConfig struct { // MaxSize is the max size per glob before compression MaxSize int // MaxAge is the max age of the oldest entry in the glob MaxAge string // BufferLength is the length of the channel where newly // submitted entries are stored, decreasing the size of this // buffer can cause stalls, and increasing the size can increase // shutdown time BufferLength int }
GlobberConfig is used to configure a globber instance
func (*GlobberConfig) Validate ¶
func (c *GlobberConfig) Validate() error
Validate returns an error if the config is invalid, nil otherwise.
type KinesisEventFilterConfig ¶
type KinesisEventFilterConfig struct { Field string Values []string Operator FilterOperator }
KinesisEventFilterConfig represents field/values that will be used to filter Events written to a Kinesis stream.
func (*KinesisEventFilterConfig) Match ¶
func (f *KinesisEventFilterConfig) Match(fieldValue string) bool
Match returns true if the fieldValue matches the filter condition.
type KinesisWriterConfig ¶
type KinesisWriterConfig struct { StreamName string StreamRole string StreamType string // StreamType should be either "stream" or "firehose" StreamRegion string // AWS region to write to. Blank to use default region. Compress bool // true if compress data with flate, false to output json FirehoseRedshiftStream bool // true if JSON destined for Firehose->Redshift streaming EventNameTargetField string // Field name to write the event's name to (useful for uncompressed streams) ExcludeEmptyFields bool // true if empty fields should be excluded from the JSON BufferSize int MaxAttemptsPerRecord int RetryDelay string Events map[string]*KinesisWriterEventConfig Globber GlobberConfig Batcher BatcherConfig }
KinesisWriterConfig describes a Kinesis Writer that the processor uses to export data to a Kinesis Stream/Firehose Make sure to call Validate() on Spade after loading this from JSON to populate some derived fields.
func (*KinesisWriterConfig) Validate ¶
func (c *KinesisWriterConfig) Validate(commonFilters map[string]EventFilterFunc) error
Validate returns an error if the Kinesis Writer config is not valid, or nil if it is. It also sets the FilterFunc on Events with Filters and populates FullFieldMap.
type KinesisWriterEventConfig ¶
type KinesisWriterEventConfig struct { Filter string FilterFunc EventFilterFunc `json:"-"` Fields []string FieldRenames map[string]string FullFieldMap map[string]string `json:"-"` FilterParameters []*KinesisEventFilterConfig SkipDefaultFilter bool }
KinesisWriterEventConfig describes how a given Event is written to a Kinesis stream.
type LoadCheckRequest ¶
type LoadCheckRequest struct {
ManifestURL string
}
type LoadCheckResponse ¶
type LoadCheckResponse struct { LoadStatus LoadStatus ManifestURL string }
type LoadStatus ¶
type LoadStatus string
const ( LoadNotFound LoadStatus = "load-not-found" LoadFailed LoadStatus = "load-failed" LoadInProgress LoadStatus = "load-in-progress" LoadComplete LoadStatus = "load-complete" )
type ManifestRowCopyRequest ¶
type Operation ¶
type Operation struct { Action Action Name string ActionMetadata map[string]string Version int Ordering int }
Operation represents a single change to a schema
func NewAddOperation ¶
func NewDeleteOperation ¶
func NewDropEventOperation ¶
func NewRenameOperation ¶
type RowCopyRequest ¶
type ScoopHealthCheck ¶
type ScoopSigner ¶
type TestableKinesisEventFilter ¶
type TestableKinesisEventFilter struct { Config []*KinesisEventFilterConfig MatchingEvents []map[string]string NonMatchingEvents []map[string]string }
TestableKinesisEventFilter is a KinesisEventFilterConfig with test cases.
func (*TestableKinesisEventFilter) Build ¶
func (f *TestableKinesisEventFilter) Build() (EventFilterFunc, error)
Build validates the config and then returns the generated EventFilterFunc.