Documentation

Overview

    Package output provides output components

    Index

    Constants

    This section is empty.

    Variables

      All is the list of all baker outputs.

      View Source
      var BakerTransport = http.DefaultTransport.(*http.Transport).Clone()
      View Source
      var DynamoDBDesc = baker.OutputDesc{
      	Name:   "DynamoDB",
      	New:    NewDynamoDB,
      	Config: &DynamoDBConfig{},
      	Raw:    false,
      	Help: "This output writes the filtered log lines to DynamoDB. It must be\n" +
      		"configured specifying the region, the table name, and the columns\n" +
      		"to write.\nColumns are specified using the syntax \"t:name\" where \"t\"\n" +
      		"is the type of the data, and \"name\" is the name of column. Supported\n" +
      		"types are: \"n\" - integers; \"s\" - strings.\n" +
      		"The first column (and field) must be the primary key.\n",
      }
      View Source
      var DynamoGlobals dynamoGlobals
      View Source
      var FileWriterDesc = baker.OutputDesc{
      	Name:   "FileWriter",
      	New:    NewFileWriter,
      	Config: &FileWriterConfig{},
      	Raw:    true,
      	Help:   helpMsg,
      }
      View Source
      var NopDesc = baker.OutputDesc{
      	Name:   "Nop",
      	New:    NewNop,
      	Config: &NopConfig{},
      	Help:   "No-operation output. This output simply drops all lines and does not write them anywhere.",
      }
      View Source
      var OpLogDesc = baker.OutputDesc{
      	Name:   "OpLog",
      	New:    NewOpLog,
      	Config: &OpLogConfig{},
      	Raw:    false,
      	Help:   "This output writes the filtered log lines into the current baker log, purely for development purpose.\n",
      }
      View Source
      var SQLiteDesc = baker.OutputDesc{
      	Name:   "SQLite",
      	New:    NewSQLite(false),
      	Config: &SQLiteConfig{},
      	Help:   "Writes a chosen set of fields as table columns into a local SQLite database file",
      	Raw:    false,
      }

        SQLiteDesc declares the standard (non-raw) SQLite output.

        View Source
        var SQLiteRawDesc = baker.OutputDesc{
        	Name:   "SQLiteRaw",
        	New:    NewSQLite(true),
        	Config: &SQLiteRawWriterConfig{},
        	Help:   "Writes a chosen set of fields, plus the raw record, as table columns into a local SQLite database file",
        	Raw:    true,
        }

          SQLiteRawDesc declares the raw SQLite output.

          View Source
          var StatsDesc = baker.OutputDesc{
          	Name:   "Stats",
          	New:    NewStats,
          	Config: &StatsConfig{},
          	Raw:    true,
          	Help: "Compute various distributions of the records it " +
          		"receives and dumps that to CSV. It computes the " +
          		"distribution of record by size and the distribution " +
          		"of the values of certain fields\n",
          }
          View Source
          var WebSocketDesc = baker.OutputDesc{
          	Name:   "WebSocket",
          	New:    NewWebSocket,
          	Config: &WebSocketConfig{},
          	Raw:    false,
          	Help:   "This output writes the filtered log lines into any connected WebSocket client.\n",
          }

          Functions

          func NewDynamoDB

          func NewDynamoDB(cfg baker.OutputParams) (baker.Output, error)

            NewDynamoDB create a new DynamoDB output.

            TableName is the name of the DynamoDB table to be written. Columns is a slice listing the columns that will be written; the first item in the slice *MUST* be the primary key of the table.

            func NewFileWriter

            func NewFileWriter(cfg baker.OutputParams) (baker.Output, error)

            func NewNop

            func NewNop(cfg baker.OutputParams) (baker.Output, error)

            func NewOpLog

            func NewOpLog(cfg baker.OutputParams) (baker.Output, error)

            func NewSQLite

            func NewSQLite(isRaw bool) func(baker.OutputParams) (baker.Output, error)

            func NewStats

            func NewStats(cfg baker.OutputParams) (baker.Output, error)

              NewStats returns a new Stats Baker output.

              func NewWebSocket

              func NewWebSocket(cfg baker.OutputParams) (baker.Output, error)

              Types

              type DynamoDB

              type DynamoDB struct {
              	TableName string
              	Fields    []baker.FieldIndex
              	Columns   []string
              	Cfg       *DynamoDBConfig
              	// contains filtered or unexported fields
              }

                DynamoDB is a class to do optimized batched writes to a single DynamoDB table with a fixed schema (same number of columns for all records).

                func (*DynamoDB) CanShard

                func (b *DynamoDB) CanShard() bool

                func (*DynamoDB) Flush

                func (b *DynamoDB) Flush()

                func (*DynamoDB) NumProcessedRecords

                func (b *DynamoDB) NumProcessedRecords() int64

                func (*DynamoDB) Run

                func (b *DynamoDB) Run(input <-chan baker.OutputRecord, _ chan<- string) error

                func (*DynamoDB) Stats

                func (b *DynamoDB) Stats() baker.OutputStats

                type DynamoDBConfig

                type DynamoDBConfig struct {
                	Regions         []string      `help:"DynamoDB regions to connect to" default:"us-west-2"`
                	Table           string        `help:"Name of the table to modify" required:"true"`
                	Columns         []string      `help:"Table columns that correspond to each of the fields being written"`
                	FlushInterval   time.Duration `help:"Interval at which flush the data to DynamoDB even if we have not reached 25 records" default:"1s"`
                	MaxWritesPerSec int64         `help:"Maximum number of writes per second that DynamoDB can accept (0 for unlimited)" default:"0"`
                	MaxBackoff      time.Duration `help:"Maximum retry/backoff time in case of errors before giving up" default:"2m"`
                	// contains filtered or unexported fields
                }

                type FileWriter

                type FileWriter struct {
                	Cfg *FileWriterConfig
                
                	Fields []baker.FieldIndex
                	// contains filtered or unexported fields
                }

                func (*FileWriter) CanShard

                func (w *FileWriter) CanShard() bool

                func (*FileWriter) Run

                func (w *FileWriter) Run(input <-chan baker.OutputRecord, upch chan<- string) error

                func (*FileWriter) Stats

                func (w *FileWriter) Stats() baker.OutputStats

                type FileWriterConfig

                type FileWriterConfig struct {
                	PathString           string        `` /* 182-byte string literal not displayed */
                	RotateInterval       time.Duration `help:"Time after which data will be rotated. If -1, it will not rotate until the end." default:"60s"`
                	ZstdCompressionLevel int           `help:"zstd compression level, ranging from 1 (best speed) to 19 (best compression)." default:"3"`
                	ZstdWindowLog        int           `` /* 183-byte string literal not displayed */
                }

                type Nop

                type Nop struct {
                	// contains filtered or unexported fields
                }

                func (*Nop) CanShard

                func (b *Nop) CanShard() bool

                func (*Nop) Run

                func (nop *Nop) Run(input <-chan baker.OutputRecord, upch chan<- string) error

                func (*Nop) Stats

                func (nop *Nop) Stats() baker.OutputStats

                type NopConfig

                type NopConfig struct{}

                type OpLog

                type OpLog struct {
                	Cfg *OpLogConfig
                
                	Fields []baker.FieldIndex
                	// contains filtered or unexported fields
                }

                func (*OpLog) CanShard

                func (b *OpLog) CanShard() bool

                func (*OpLog) Run

                func (w *OpLog) Run(input <-chan baker.OutputRecord, _ chan<- string) error

                func (*OpLog) Stats

                func (w *OpLog) Stats() baker.OutputStats

                type OpLogConfig

                type OpLogConfig struct{}

                type SQLite

                type SQLite struct {
                	// contains filtered or unexported fields
                }

                func (*SQLite) CanShard

                func (c *SQLite) CanShard() bool

                func (*SQLite) Run

                func (c *SQLite) Run(input <-chan baker.OutputRecord, upch chan<- string) error

                func (*SQLite) Stats

                func (c *SQLite) Stats() baker.OutputStats

                type SQLiteConfig

                type SQLiteConfig struct {
                	PathString string   `` /* 167-byte string literal not displayed */
                	TableName  string   `help:"Table name to which to write the records to." required:"true"`
                	PreRun     []string `help:"List of SQL statements to run at startup (before table creation)."`
                	PostRun    []string `help:"List of SQL statements to run at exit. (good place to create indexes if needed)."`
                	Clear      bool     `` /* 233-byte string literal not displayed */
                	Vacuum     bool     `` /* 259-byte string literal not displayed */
                	Wal        bool     `` /* 167-byte string literal not displayed */
                	PageSize   int64    `help:"The page size to use for SQLite. By default, we use whatever SQLite decides to use as default."`
                }

                  SQLiteConfig holds the configuration parameters for the standard SQLite baker output.

                  type SQLiteRawWriterConfig

                  type SQLiteRawWriterConfig struct {
                  	PathString     string   `` /* 167-byte string literal not displayed */
                  	TableName      string   `help:"Table name to which to write the records to." required:"true"`
                  	PreRun         []string `help:"List of SQL statements to run at startup (before table creation)."`
                  	PostRun        []string `help:"List of SQL statements to run at exit. (good place to create indexes if needed)."`
                  	Clear          bool     `` /* 233-byte string literal not displayed */
                  	Vacuum         bool     `` /* 259-byte string literal not displayed */
                  	Wal            bool     `` /* 167-byte string literal not displayed */
                  	PageSize       int64    `help:"The page size to use for SQLite. By default, we use whatever SQLite decides to use as default."`
                  	RecordBlobName string   `help:"Name of the column in which the whole raw record should be put." required:"true"`
                  }

                    SQLiteRawWriterConfig holds the configuration parameters for the raw SQLite baker output.

                    type Stats

                    type Stats struct {
                    	// contains filtered or unexported fields
                    }

                      The Stats output gathers statistics about the sizes of log lines it sees.

                      func (*Stats) CanShard

                      func (s *Stats) CanShard() bool

                        CanShard implements baker.Output

                        func (*Stats) Run

                        func (s *Stats) Run(input <-chan baker.OutputRecord, _ chan<- string) error

                          Run implements baker.Output

                          func (*Stats) Stats

                          func (s *Stats) Stats() baker.OutputStats

                            Stats implements baker.Output

                            type StatsConfig

                            type StatsConfig struct {
                            	CountEmptyFields bool   `help:"Whether fields with empty values are counted or not" default:"false"`
                            	CSVPath          string `help:"Path of the CSV file to create" default:"stats.csv"`
                            	TimestampField   string `help:"Name of a field containing a POSIX timestamp (in seconds) used to build the times stats" required:"true"`
                            }

                            type WebSocket

                            type WebSocket struct {
                            	Cfg *WebSocketConfig
                            
                            	Fields []baker.FieldIndex
                            	// contains filtered or unexported fields
                            }

                            func (*WebSocket) CanShard

                            func (b *WebSocket) CanShard() bool

                            func (*WebSocket) Run

                            func (w *WebSocket) Run(input <-chan baker.OutputRecord, _ chan<- string) error

                            func (*WebSocket) Stats

                            func (w *WebSocket) Stats() baker.OutputStats

                            type WebSocketConfig

                            type WebSocketConfig struct{}

                            Directories

                            Path Synopsis