Versions in this module Expand all Collapse all v4 v4.14.1 Apr 26, 2023 Changes in this version + const GCPCloudStorageAppendCollisionMode + const GCPCloudStorageErrorIfExistsCollisionMode + const GCPCloudStorageIgnoreCollisionMode + const GCPCloudStorageOverwriteCollisionMode + var InjectTracingSpanMappingDocs = ...() + func Description(async, batches bool, content string) string + func IterateBatchedSend(msg message.Batch, fn func(int, *message.Part) error) error + type AMQP1Config struct + ApplicationPropertiesMapping string + MaxInFlight int + Metadata metadata.ExcludeFilterConfig + SASL shared.SASLConfig + TLS btls.Config + TargetAddress string + URL string + func NewAMQP1Config() AMQP1Config + type AMQPConfig struct + BindingKey string + ContentEncoding string + ContentType string + Exchange string + ExchangeDeclare AMQPExchangeDeclareConfig + Immediate bool + Mandatory bool + MaxInFlight int + Metadata metadata.ExcludeFilterConfig + Persistent bool + Priority string + TLS btls.Config + Timeout string + Type string + URLs []string + func NewAMQPConfig() AMQPConfig + type AMQPExchangeDeclareConfig struct + Durable bool + Enabled bool + Type string + type AmazonS3Config struct + Batching batchconfig.Config + Bucket string + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + ForcePathStyleURLs bool + KMSKeyID string + MaxInFlight int + Metadata metadata.ExcludeFilterConfig + Path string + ServerSideEncryption string + StorageClass string + Tags map[string]string + Timeout string + WebsiteRedirectLocation string + func NewAmazonS3Config() AmazonS3Config + type AmazonSQSConfig struct + Batching batchconfig.Config + MaxInFlight int + MessageDeduplicationID string + MessageGroupID string + Metadata metadata.ExcludeFilterConfig + URL string + func NewAmazonSQSConfig() AmazonSQSConfig + type AsyncSink interface + Close func(ctx context.Context) error + Connect func(ctx context.Context) error + WriteBatch func(ctx context.Context, msg message.Batch) error + type AsyncWriter struct + func (w *AsyncWriter) Connected() bool + func (w *AsyncWriter) Consume(ts <-chan message.Transaction) error + func (w *AsyncWriter) SetInjectTracingMap(exec *mapping.Executor) + func (w *AsyncWriter) TriggerCloseNow() + func (w *AsyncWriter) WaitForClose(ctx context.Context) error + type AzureBlobStorageConfig struct + BlobType string + Container string + MaxInFlight int + Path string + PublicAccessLevel string + StorageAccessKey string + StorageAccount string + StorageConnectionString string + StorageSASToken string + func NewAzureBlobStorageConfig() AzureBlobStorageConfig + type AzureQueueStorageConfig struct + Batching batchconfig.Config + MaxInFlight int + QueueName string + StorageAccessKey string + StorageAccount string + StorageConnectionString string + TTL string + func NewAzureQueueStorageConfig() AzureQueueStorageConfig + type AzureTableStorageConfig struct + Batching batchconfig.Config + InsertType string + MaxInFlight int + PartitionKey string + Properties map[string]string + RowKey string + StorageAccessKey string + StorageAccount string + StorageConnectionString string + TableName string + Timeout string + TransactionType string + func NewAzureTableStorageConfig() AzureTableStorageConfig + type BrokerConfig struct + Batching batchconfig.Config + Copies int + Outputs []Config + Pattern string + func NewBrokerConfig() BrokerConfig + type CacheConfig struct + Key string + MaxInFlight int + TTL string + Target string + func NewCacheConfig() CacheConfig + type CassandraConfig struct + Addresses []string + ArgsMapping string + Batching batchconfig.Config + Consistency string + DisableInitialHostLookup bool + LoggedBatch bool + MaxInFlight int + PasswordAuthenticator PasswordAuthenticator + Query string + TLS btls.Config + Timeout string + func NewCassandraConfig() CassandraConfig + type Config struct + AMQP09 AMQPConfig + AMQP1 AMQP1Config + AWSDynamoDB DynamoDBConfig + AWSKinesis KinesisConfig + AWSKinesisFirehose KinesisFirehoseConfig + AWSS3 AmazonS3Config + AWSSNS SNSConfig + AWSSQS AmazonSQSConfig + AzureBlobStorage AzureBlobStorageConfig + AzureQueueStorage AzureQueueStorageConfig + AzureTableStorage AzureTableStorageConfig + Broker BrokerConfig + Cache CacheConfig + Cassandra CassandraConfig + Drop DropConfig + DropOn DropOnConfig + Dynamic DynamicConfig + Elasticsearch ElasticsearchConfig + Fallback TryConfig + File FileConfig + GCPCloudStorage GCPCloudStorageConfig + GCPPubSub GCPPubSubConfig + HDFS HDFSConfig + HTTPServer HTTPServerConfig + Inproc string + Kafka KafkaConfig + Label string + MQTT MQTTConfig + MongoDB MongoDBConfig + NATS NATSConfig + NATSStream NATSStreamConfig + NSQ NSQConfig + Nanomsg NanomsgConfig + Plugin any + Processors []processor.Config + RedisHash RedisHashConfig + RedisList RedisListConfig + RedisPubSub RedisPubSubConfig + RedisStreams RedisStreamsConfig + Reject string + Resource string + Retry RetryConfig + SFTP SFTPConfig + STDOUT STDOUTConfig + Socket SocketConfig + Subprocess SubprocessConfig + Switch SwitchConfig + SyncResponse struct{} + Type string + func NewConfig() Config + func (conf *Config) UnmarshalYAML(value *yaml.Node) error + type DropConfig struct + func NewDropConfig() DropConfig + type DropOnConditions struct + BackPressure string + Error bool + type DropOnConfig struct + Output *Config + func NewDropOnConfig() DropOnConfig + func (d DropOnConfig) MarshalJSON() ([]byte, error) + func (d DropOnConfig) MarshalYAML() (any, error) + type DynamicConfig struct + Outputs map[string]Config + Prefix string + func NewDynamicConfig() DynamicConfig + type DynamoDBConfig struct + Batching batchconfig.Config + JSONMapColumns map[string]string + MaxInFlight int + StringColumns map[string]string + TTL string + TTLKey string + Table string + func NewDynamoDBConfig() DynamoDBConfig + type ElasticsearchAuthConfig struct + Enabled bool + Password string + Username string + type ElasticsearchConfig struct + AWS OptionalAWSConfig + Action string + Auth ElasticsearchAuthConfig + Batching batchconfig.Config + GzipCompression bool + Healthcheck bool + ID string + Index string + MaxInFlight int + Pipeline string + Routing string + Sniff bool + TLS btls.Config + Timeout string + Type string + URLs []string + func NewElasticsearchConfig() ElasticsearchConfig + type FileConfig struct + Codec string + Path string + func NewFileConfig() FileConfig + type GCPCloudStorageConfig struct + Batching batchconfig.Config + Bucket string + ChunkSize int + CollisionMode string + ContentEncoding string + ContentType string + MaxInFlight int + Path string + func NewGCPCloudStorageConfig() GCPCloudStorageConfig + type GCPPubSubConfig struct + Endpoint string + FlowControl GCPPubSubFlowControlConfig + MaxInFlight int + Metadata metadata.ExcludeFilterConfig + OrderingKey string + ProjectID string + PublishTimeout string + TopicID string + func NewGCPPubSubConfig() GCPPubSubConfig + type GCPPubSubFlowControlConfig struct + LimitExceededBehavior string + MaxOutstandingBytes int + MaxOutstandingMessages int + func NewGCPPubSubFlowControlConfig() GCPPubSubFlowControlConfig + type HDFSConfig struct + Batching batchconfig.Config + Directory string + Hosts []string + MaxInFlight int + Path string + User string + func NewHDFSConfig() HDFSConfig + type HTTPServerConfig struct + Address string + AllowedVerbs []string + CORS httpserver.CORSConfig + CertFile string + KeyFile string + Path string + StreamPath string + Timeout string + WSPath string + func NewHTTPServerConfig() HTTPServerConfig + type KafkaConfig struct + AckReplicas bool + Addresses []string + Batching batchconfig.Config + ClientID string + Compression string + InjectTracingMap string + Key string + MaxInFlight int + MaxMsgBytes int + Metadata metadata.ExcludeFilterConfig + Partition string + Partitioner string + RackID string + RetryAsBatch bool + SASL sasl.Config + StaticHeaders map[string]string + TLS btls.Config + TargetVersion string + Timeout string + Topic string + func NewKafkaConfig() KafkaConfig + type KinesisConfig struct + Batching batchconfig.Config + HashKey string + MaxInFlight int + PartitionKey string + Stream string + func NewKinesisConfig() KinesisConfig + type KinesisFirehoseConfig struct + Batching batchconfig.Config + MaxInFlight int + Stream string + func NewKinesisFirehoseConfig() KinesisFirehoseConfig + type MQTTConfig struct + ClientID string + ConnectTimeout string + DynamicClientIDSuffix string + KeepAlive int64 + MaxInFlight int + Password string + QoS uint8 + Retained bool + RetainedInterpolated string + TLS tls.Config + Topic string + URLs []string + User string + Will mqttconf.Will + WriteTimeout string + func NewMQTTConfig() MQTTConfig + type MongoDBConfig struct + Batching batchconfig.Config + DocumentMap string + FilterMap string + HintMap string + MaxInFlight int + MongoConfig client.Config + Operation string + RetryConfig retries.Config + Upsert bool + WriteConcern client.WriteConcern + func NewMongoDBConfig() MongoDBConfig + type NATSConfig struct + Auth auth.Config + Headers map[string]string + MaxInFlight int + Subject string + TLS btls.Config + URLs []string + func NewNATSConfig() NATSConfig + type NATSStreamConfig struct + Auth auth.Config + ClientID string + ClusterID string + MaxInFlight int + Subject string + TLS btls.Config + URLs []string + func NewNATSStreamConfig() NATSStreamConfig + type NSQConfig struct + Address string + MaxInFlight int + TLS btls.Config + Topic string + UserAgent string + func NewNSQConfig() NSQConfig + type NanomsgConfig struct + Bind bool + MaxInFlight int + PollTimeout string + SocketType string + URLs []string + func NewNanomsgConfig() NanomsgConfig + type OptionalAWSConfig struct + Enabled bool + type PasswordAuthenticator struct + Enabled bool + Password string + Username string + type RedisHashConfig struct + Fields map[string]string + Key string + MaxInFlight int + WalkJSONObject bool + WalkMetadata bool + func NewRedisHashConfig() RedisHashConfig + type RedisListConfig struct + Batching batchconfig.Config + Key string + MaxInFlight int + func NewRedisListConfig() RedisListConfig + type RedisPubSubConfig struct + Batching batchconfig.Config + Channel string + MaxInFlight int + func NewRedisPubSubConfig() RedisPubSubConfig + type RedisStreamsConfig struct + Batching batchconfig.Config + BodyKey string + MaxInFlight int + MaxLenApprox int64 + Metadata metadata.ExcludeFilterConfig + Stream string + func NewRedisStreamsConfig() RedisStreamsConfig + type RetryConfig struct + Output *Config + func NewRetryConfig() RetryConfig + func (r RetryConfig) MarshalJSON() ([]byte, error) + func (r RetryConfig) MarshalYAML() (any, error) + type SFTPConfig struct + Address string + Codec string + Credentials sftpSetup.Credentials + MaxInFlight int + Path string + func NewSFTPConfig() SFTPConfig + type SNSConfig struct + MaxInFlight int + MessageDeduplicationID string + MessageGroupID string + Metadata metadata.ExcludeFilterConfig + Timeout string + TopicArn string + func NewSNSConfig() SNSConfig + type STDOUTConfig struct + Codec string + func NewSTDOUTConfig() STDOUTConfig + type SessionConfig struct + type SocketConfig struct + Address string + Codec string + Network string + func NewSocketConfig() SocketConfig + type Streamed interface + Connected func() bool + Consume func(<-chan message.Transaction) error + TriggerCloseNow func() + WaitForClose func(ctx context.Context) error + func NewAsyncWriter(typeStr string, maxInflight int, w AsyncSink, mgr component.Observability) (Streamed, error) + func OnlySinglePayloads(out Streamed) Streamed + func WrapWithPipelines(out Streamed, pipeConstructors ...iprocessor.PipelineConstructorFunc) (Streamed, error) + type SubprocessConfig struct + Args []string + Codec string + Name string + func NewSubprocessConfig() SubprocessConfig + type SwitchConfig struct + Cases []SwitchConfigCase + RetryUntilSuccess bool + StrictMode bool + func NewSwitchConfig() SwitchConfig + type SwitchConfigCase struct + Check string + Continue bool + Output Config + func NewSwitchConfigCase() SwitchConfigCase + type Sync interface + Connected func() bool + TriggerCloseNow func() + TriggerStopConsuming func() + WaitForClose func(ctx context.Context) error + WriteTransaction func(context.Context, message.Transaction) error + type TryConfig []Config + func NewTryConfig() TryConfig + type WithPipeline struct + func WrapWithPipeline(out Streamed, pipeConstructor iprocessor.PipelineConstructorFunc) (*WithPipeline, error) + func (i *WithPipeline) Connected() bool + func (i *WithPipeline) Consume(tsChan <-chan message.Transaction) error + func (i *WithPipeline) TriggerCloseNow() + func (i *WithPipeline) WaitForClose(ctx context.Context) error Other modules containing this package github.com/dafanshu/benthos/v3