destination

Documentation

Index

• func PluginTestSuiteRunner(t *testing.T, p *Plugin, spec any, tests PluginTestSuiteTests)
• func SetDestinationManagedCqColumns(tables []*schema.Table)
• type Client
• type ClientResource
• type DefaultReverseTransformer
  • func (*DefaultReverseTransformer) ReverseTransformValues(table *schema.Table, values []any) (schema.CQTypes, error)
• type ManagedWriter
• type Metrics
• type NewClientFunc
• type Option
  • func WithBatchTimeout(seconds int) Option
  • func WithManagedWriter() Option
• type Plugin
  • func NewPlugin(name string, version string, newClientFunc NewClientFunc, opts ...Option) *Plugin
  • func (p *Plugin) Close(ctx context.Context) error
  • func (p *Plugin) DeleteStale(ctx context.Context, tables schema.Tables, sourceName string, ...) error
  • func (p *Plugin) Init(ctx context.Context, logger zerolog.Logger, spec specs.Destination) error
  • func (p *Plugin) Metrics() Metrics
  • func (p *Plugin) Migrate(ctx context.Context, tables schema.Tables) error
  • func (p *Plugin) Name() string
  • func (p *Plugin) Read(ctx context.Context, table *schema.Table, sourceName string, ...) error
  • func (p *Plugin) Version() string
  • func (p *Plugin) Write(ctx context.Context, tables schema.Tables, sourceName string, ...) error
• type PluginTestSuite
• type PluginTestSuiteTests
• type UnimplementedManagedWriter
  • func (*UnimplementedManagedWriter) WriteTableBatch(context.Context, *schema.Table, [][]any) error
• type UnimplementedUnmanagedWriter
  • func (*UnimplementedUnmanagedWriter) Metrics() Metrics
  • func (*UnimplementedUnmanagedWriter) Write(context.Context, schema.Tables, <-chan *ClientResource) error
• type UnmanagedWriter

Functions

func PluginTestSuiteRunner

func PluginTestSuiteRunner(t *testing.T, p *Plugin, spec any, tests PluginTestSuiteTests)

func SetDestinationManagedCqColumns

func SetDestinationManagedCqColumns(tables []*schema.Table)

Overwrites or adds the CQ columns that are managed by the destination plugins (_cq_sync_time, _cq_source_name).

Types

type Client

type Client interface {
	schema.CQTypeTransformer
	ReverseTransformValues(table *schema.Table, values []any) (schema.CQTypes, error)
	Migrate(ctx context.Context, tables schema.Tables) error
	Read(ctx context.Context, table *schema.Table, sourceName string, res chan<- []any) error
	ManagedWriter
	UnmanagedWriter
	DeleteStale(ctx context.Context, tables schema.Tables, sourceName string, syncTime time.Time) error
	Close(ctx context.Context) error
}

type ClientResource

type ClientResource struct {
	TableName string
	Data      []any
}

type DefaultReverseTransformer

type DefaultReverseTransformer struct {
}

func (*DefaultReverseTransformer) ReverseTransformValues

func (*DefaultReverseTransformer) ReverseTransformValues(table *schema.Table, values []any) (schema.CQTypes, error)

DefaultReverseTransformer tries best effort to convert a slice of values to CQTypes based on the provided table columns.

type ManagedWriter

type ManagedWriter interface {
	WriteTableBatch(ctx context.Context, table *schema.Table, data [][]any) error
}

type Metrics

type Metrics struct {
	// Errors number of errors / failed writes
	Errors uint64
	// Writes number of successful writes
	Writes uint64
}

type NewClientFunc

type NewClientFunc func(context.Context, zerolog.Logger, specs.Destination) (Client, error)

type Option

type Option func(*Plugin)

func WithBatchTimeout

func WithBatchTimeout(seconds int) Option

func WithManagedWriter

func WithManagedWriter() Option

type Plugin

type Plugin struct {
	// contains filtered or unexported fields
}

func NewPlugin

func NewPlugin(name string, version string, newClientFunc NewClientFunc, opts ...Option) *Plugin

NewPlugin creates a new destination plugin

func (*Plugin) Close

func (p *Plugin) Close(ctx context.Context) error

func (*Plugin) DeleteStale

func (p *Plugin) DeleteStale(ctx context.Context, tables schema.Tables, sourceName string, syncTime time.Time) error

func (*Plugin) Init

func (p *Plugin) Init(ctx context.Context, logger zerolog.Logger, spec specs.Destination) error

we need lazy loading because we want to be able to initialize after

func (*Plugin) Metrics

func (p *Plugin) Metrics() Metrics

func (*Plugin) Migrate

func (p *Plugin) Migrate(ctx context.Context, tables schema.Tables) error

we implement all DestinationClient functions so we can hook into pre-post behavior

func (*Plugin) Name

func (p *Plugin) Name() string

func (*Plugin) Read

func (p *Plugin) Read(ctx context.Context, table *schema.Table, sourceName string, res chan<- schema.CQTypes) error

func (*Plugin) Version

func (p *Plugin) Version() string

func (*Plugin) Write

func (p *Plugin) Write(ctx context.Context, tables schema.Tables, sourceName string, syncTime time.Time, res <-chan schema.DestinationResource) error

type PluginTestSuite

type PluginTestSuite struct {
	// contains filtered or unexported fields
}

type PluginTestSuiteTests

type PluginTestSuiteTests struct {
	// SkipOverwrite skips testing for "overwrite" mode. Use if the destination
	//	// plugin doesn't support this feature.
	SkipOverwrite bool

	// SkipDeleteStale skips testing "delete-stale" mode. Use if the destination
	// plugin doesn't support this feature.
	SkipDeleteStale bool

	// SkipAppend skips testing for "append" mode. Use if the destination
	// plugin doesn't support this feature.
	SkipAppend bool

	// SkipSecondAppend skips the second append step in the test.
	// This is useful in cases like cloud storage where you can't append to an
	// existing object after the file has been closed.
	SkipSecondAppend bool
}

type UnimplementedManagedWriter

type UnimplementedManagedWriter struct{}

func (*UnimplementedManagedWriter) WriteTableBatch

func (*UnimplementedManagedWriter) WriteTableBatch(context.Context, *schema.Table, [][]any) error

type UnimplementedUnmanagedWriter

type UnimplementedUnmanagedWriter struct{}

func (*UnimplementedUnmanagedWriter) Metrics

func (*UnimplementedUnmanagedWriter) Metrics() Metrics

func (*UnimplementedUnmanagedWriter) Write

func (*UnimplementedUnmanagedWriter) Write(context.Context, schema.Tables, <-chan *ClientResource) error

type UnmanagedWriter

type UnmanagedWriter interface {
	Write(ctx context.Context, tables schema.Tables, res <-chan *ClientResource) error
	Metrics() Metrics
}