config

Documentation

Index

• Variables
• func InitS3StreamingLister(app *S3StreamingLister)
• func ParseArgs(app *S3StreamingLister, osArgs []string) error
• type AwsParams
• type CallStat
• type Calls
  • func MakeCalls() *Calls
  • func (calls *Calls) Add(add int, keys ...string)
  • func (calls *Calls) Dec(keys ...string)
  • func (calls *Calls) Duration(key string, start time.Time)
  • func (calls *Calls) Get(key string) CallStat
  • func (calls *Calls) Inc(keys ...string)
  • func (calls *Calls) Keys(keys ...string) []string
• type Channels
• type Config
• type Counter
• type DynamoDbParams
• type FrontendParams
• type LambdaParams
• type ListObjectParams
• type MyCredentials
  • func (my *MyCredentials) Retrieve(ctx context.Context) (aws.Credentials, error)
• type Output
• type OutputParams
• type ParquetFrontendParams
• type ParquetParams
• type S3DeleteParams
• type S3StreamingLister
  • func DefaultS3StreamingLister() *S3StreamingLister
• type SqliteFrontendParams
• type SqliteParams
• type SqsParams

Variables

var GitCommit string

var Version string

Functions

func InitS3StreamingLister

func InitS3StreamingLister(app *S3StreamingLister)

func ParseArgs

func ParseArgs(app *S3StreamingLister, osArgs []string) error

Types

type AwsParams

type AwsParams struct {
	KeyId           *string
	SecretAccessKey *string
	SessionToken    *string
	Region          *string
	Cfg             aws.Config
}

type CallStat

type CallStat struct {
	Cnt      int64
	Duration int64
}

type Calls

type Calls struct {
	// contains filtered or unexported fields
}

func MakeCalls

func MakeCalls() *Calls

func (*Calls) Add

func (calls *Calls) Add(add int, keys ...string)

func (*Calls) Dec

func (calls *Calls) Dec(keys ...string)

func (*Calls) Duration

func (calls *Calls) Duration(key string, start time.Time)

func (*Calls) Get

func (calls *Calls) Get(key string) CallStat

func (*Calls) Inc

func (calls *Calls) Inc(keys ...string)

func (*Calls) Keys

func (calls *Calls) Keys(keys ...string) []string

type Channels

type Channels struct {
	Calls    Counter
	Channels chan *s3.Client
}

type Config

type Config struct {
	GitCommit     string
	Version       string
	Strategy      *string
	Prefixes      *[]string
	Prefix        *string
	Delimiter     *string
	Format        *string
	Bucket        *string
	MaxKeys       *int
	S3Workers     *int
	StatsFragment *uint64
	Help          bool
	VersionFlag   bool
	Progress      *int
	Output        OutputParams
	ListObject    ListObjectParams
	Lambda        LambdaParams
	Frontend      FrontendParams
}

type Counter

type Counter struct {
	Total      Calls
	Concurrent Calls
	Error      Calls
}

type DynamoDbParams

type DynamoDbParams struct {
	Workers *int
	Aws     AwsParams
}

type FrontendParams

type FrontendParams struct {
	Sqlite   SqliteFrontendParams
	Parquet  ParquetFrontendParams
	Frontend *string
}

type LambdaParams

type LambdaParams struct {
	Deploy *bool
	Start  *bool
	Aws    AwsParams
}

type ListObjectParams

type ListObjectParams struct {
	Aws AwsParams
}

type MyCredentials

type MyCredentials struct {
	// contains filtered or unexported fields
}

func (*MyCredentials) Retrieve

func (my *MyCredentials) Retrieve(ctx context.Context) (aws.Credentials, error)

type Output

type Output struct {
	FileStream io.Writer
}

type OutputParams

type OutputParams struct {
	Sqs      SqsParams
	Sqlite   SqliteParams
	Parquet  ParquetParams
	S3Delete S3DeleteParams
	DynamoDb DynamoDbParams
}

type ParquetFrontendParams

type ParquetFrontendParams struct {
	Filename  *string
	Workers   *int
	RowBuffer *int
}

type ParquetParams

type ParquetParams struct {
	Workers   *int
	ChunkSize *int
	FileName  *string
}

type S3DeleteParams

type S3DeleteParams struct {
	Workers   *int
	ChunkSize *int
	Aws       AwsParams
}

type S3StreamingLister

type S3StreamingLister struct {
	Config          Config
	InputConcurrent int32
	Clients         Channels
	Output          Output
}

func DefaultS3StreamingLister

func DefaultS3StreamingLister() *S3StreamingLister

type SqliteFrontendParams

type SqliteFrontendParams struct {
	Filename  *string
	Query     *string
	TableName *string
}

type SqliteParams

type SqliteParams struct {
	Workers    *int
	CommitSize *int
	CleanDb    *bool
	Filename   *string
	SqlTable   *string
}

type SqsParams

type SqsParams struct {
	Workers        *int
	ChunkSize      *int
	Url            *string
	Delay          *int32
	MaxMessageSize *int
	Aws            AwsParams
}