awstee

README

awstee

awstee is a tee command-like tool with AWS as the output destination.

The awstee command reads from standard input and writes to standard output and AWS S3 and CloudWatch Logs. awstee is the util tool for one time script for mission critical (especially for preventing rerunning it).

Usage

Basically, it can be used as follows

$ your_command |  awstee -s3-url-prefix s3://awstee-example-com/logs/  -log-group-name /awstee/logs hoge.log
2022/06/03 17:28:48 [info] s3 destination:  s3://awstee-example-com/logs//hoge.log
2022/06/03 17:28:49 [info] cloudwatch logs destination:  LogGroup=/awstee/test, LogStream=hoge
...

with default config ~/.config/awstee/default.yaml or ~/.config/awstee/default.yml.

aws_region: "ap-northeast-1"

s3:
  url_prefix: "s3://awstee-example-com/logs/" # Required if used. If blank, output setting is turned off
  allow_overwrite: true # Whether to allow overwriting if the object already exists

cloudwatch:
  log_group: "/awstee/logs" # Required if used. If blank, output setting is turned off
  flush_interval: "5s" # Duration of buffer flush output to cloudwatch logs
  buffer_lines: 50 # If more than this number of lines are output within the flush period, it is output once to Cloudwatch logs.
  create_log_group: true # Whether to create a LogGroup if it does not exist

$ your_command |  awstee hoge.log
2022/06/03 17:28:48 [info] s3 destination:  s3://awstee-example-com/logs//hoge.log
2022/06/03 17:28:49 [info] cloudwatch logs destination:  LogGroup=/awstee/test, LogStream=hoge
...

Install

Homebrew (macOS and Linux)

$ brew install mashiike/tap/awstee

Binary packages

Releases

Options

$ awstee -h    
awstee is a tee command-like tool with AWS as the output destination
version: v0.3.0 
  -aws-region string
        aws region
  -buffer-lines int
        cloudwatch logs output buffered lines (default 50)
  -config string
        config file path
  -create-log-group
        cloudwatch logs log group if not exists, create target log group
  -flush-interval string
        cloudwatch logs output flush interval duration (default "5s")
  -i    ignore interrupt signal
  -log-group-name string
        destination cloudwatch logs log group name
  -log-level string
        awstee log level (default "info")
  -s3-allow-overwrite
        allow overwriting if the s3 object already exists?
  -s3-firstly-put-empty-object
        put object from first for authority checks, etc.
  -s3-url-prefix string
        destination s3 url prefix
  -x    exit if an error occurs during initialization

IAM Role Policy

Permissions that awstee may have access to are as follows

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "S3Access",
            "Effect": "Allow",
            "Action": [
                "s3:PutObject",
                "s3:GetObject",
                "s3:AbortMultipartUpload",
                "s3:ListBucket"
            ],
            "Resource": "*"
        },
        {
            "Sid": "CloudwatchLogsAccess",
            "Effect": "Allow",
            "Action": [
                "logs:CreateLogStream",
                "logs:DescribeLogStreams",
                "logs:CreateLogGroup",
                "logs:PutLogEvents"
            ],
            "Resource": "*"
        }
    ]
}

Note: logs:CreateLogGroup privilege is used only when the -create-log-group option is enabled.

LICENSE

MIT License

Copyright (c) 2022 IKEDA Masashi

Documentation

Index

• type AWSClient
• type AWSTee
  • func New(ctx context.Context, cfg *Config) (*AWSTee, error)
  • func NewWithClient(cfg *Config, client AWSClient) (*AWSTee, error)
  • func (app *AWSTee) TeeReader(r io.Reader, outputName string) (*AWSTeeReader, error)
• type AWSTeeReader
  • func (t *AWSTeeReader) Close() error
  • func (t *AWSTeeReader) Read(p []byte) (int, error)
• type CloudwatchLogsClient
• type CloudwatchLogsConfig
  • func (cfg *CloudwatchLogsConfig) Restrict() error
  • func (cfg *CloudwatchLogsConfig) SetFlags(f *flag.FlagSet)
• type Config
  • func DefaultConfig() *Config
  • func (cfg *Config) EnableCloudwatchLogs() bool
  • func (cfg *Config) EnableS3() bool
  • func (cfg *Config) EndpointResolver() (aws.EndpointResolver, bool)
  • func (cfg *Config) Load(path string) error
  • func (cfg *Config) Restrict() error
  • func (cfg *Config) SetFlags(f *flag.FlagSet)
  • func (cfg *Config) ValidateVersion(version string) error
• type EndpointsConfig
• type S3Client
• type S3Config
  • func (cfg *S3Config) Restrict() error
  • func (cfg *S3Config) SetFlags(f *flag.FlagSet)

Types

type AWSClient

type AWSClient struct {
	S3             S3Client
	CloudwatchLogs CloudwatchLogsClient
}

type AWSTee

type AWSTee struct {
	// contains filtered or unexported fields
}

func New

func New(ctx context.Context, cfg *Config) (*AWSTee, error)

func NewWithClient

func NewWithClient(cfg *Config, client AWSClient) (*AWSTee, error)

func (*AWSTee) TeeReader

func (app *AWSTee) TeeReader(r io.Reader, outputName string) (*AWSTeeReader, error)

type AWSTeeReader

type AWSTeeReader struct {
	// contains filtered or unexported fields
}

func (*AWSTeeReader) Close

func (t *AWSTeeReader) Close() error

func (*AWSTeeReader) Read

func (t *AWSTeeReader) Read(p []byte) (int, error)

type CloudwatchLogsClient

type CloudwatchLogsClient interface {
	DescribeLogStreams(ctx context.Context, params *cloudwatchlogs.DescribeLogStreamsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error)
	PutLogEvents(ctx context.Context, input *cloudwatchlogs.PutLogEventsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.PutLogEventsOutput, error)
	CreateLogGroup(ctx context.Context, input *cloudwatchlogs.CreateLogGroupInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.CreateLogGroupOutput, error)
	CreateLogStream(ctx context.Context, input *cloudwatchlogs.CreateLogStreamInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.CreateLogStreamOutput, error)
}

type CloudwatchLogsConfig

type CloudwatchLogsConfig struct {
	LogGroup       string `yaml:"log_group,omitempty"`
	FlushInterval  string `yaml:"flush_interval,omitempty"`
	BufferLines    int    `yaml:"buffer_lines,omitempty"`
	CreateLogGroup bool   `yaml:"create_log_group,omitempty"`
	// contains filtered or unexported fields
}

func (*CloudwatchLogsConfig) Restrict

func (cfg *CloudwatchLogsConfig) Restrict() error

func (*CloudwatchLogsConfig) SetFlags

func (cfg *CloudwatchLogsConfig) SetFlags(f *flag.FlagSet)

type Config

type Config struct {
	RequiredVersion string                `yaml:"required_version,omitempty"`
	AWSRegion       string                `yaml:"aws_region,omitempty"`
	S3              *S3Config             `yaml:"s3,omitempty"`
	Cloudwatch      *CloudwatchLogsConfig `yaml:"cloudwatch,omitempty"`
	Endpoints       *EndpointsConfig      `yaml:"endpoints,omitempty"`
	// contains filtered or unexported fields
}

func DefaultConfig

func DefaultConfig() *Config

func (*Config) EnableCloudwatchLogs

func (cfg *Config) EnableCloudwatchLogs() bool

func (*Config) EnableS3

func (cfg *Config) EnableS3() bool

func (*Config) EndpointResolver

func (cfg *Config) EndpointResolver() (aws.EndpointResolver, bool)

func (*Config) Load

func (cfg *Config) Load(path string) error

func (*Config) Restrict

func (cfg *Config) Restrict() error

Restrict restricts a configuration.

func (*Config) SetFlags

func (cfg *Config) SetFlags(f *flag.FlagSet)

func (*Config) ValidateVersion

func (cfg *Config) ValidateVersion(version string) error

ValidateVersion validates a version satisfies required_version.

type EndpointsConfig

type EndpointsConfig struct {
	CloudWatchLogs string `yaml:"cloudwatchlogs,omitempty"`
	STS            string `yaml:"sts,omitempty"`
	S3             string `yaml:"s3,omitempty"`
}

type S3Client

type S3Client interface {
	s3.HeadObjectAPIClient
	manager.UploadAPIClient
}

type S3Config

type S3Config struct {
	URLPrefix             string `yaml:"url_prefix,omitempty"`
	AllowOverwrite        bool   `yaml:"allow_overwrite,omitempty"`
	FirstlyPutEmptyObject bool   `yaml:"firstly_put_empty_object,omitempty"`
	// contains filtered or unexported fields
}

func (*S3Config) Restrict

func (cfg *S3Config) Restrict() error

func (*S3Config) SetFlags

func (cfg *S3Config) SetFlags(f *flag.FlagSet)