state

Documentation

Index

• Constants
• type DownloadedObject
• type DynamoDBStater
  • func NewDynamoDBStater(session *session.Session, backfillHrs int) (*DynamoDBStater, error)
  • func (d *DynamoDBStater) ProcessedObjects() (map[string]time.Time, error)
  • func (d *DynamoDBStater) SetProcessed(s3object string) error
• type FileStater
  • func NewFileStater(stateDir, service string, backfillHrs int) *FileStater
  • func (f *FileStater) ProcessedObjects() (map[string]time.Time, error)
  • func (f *FileStater) SetProcessed(object string) error
• type Record
• type Stater

Constants

const (
	DynamoTableName = "HoneyAWSAccessLogBuckets"
	TTLDefault      = time.Hour * 24 * 7
)

Types

type DownloadedObject

type DownloadedObject struct {
	Object, Filename string
}

Used to communicate between the various pieces which are relying on state information.

type DynamoDBStater

type DynamoDBStater struct {
	Session          *session.Session
	BackfillInterval time.Duration
}

func NewDynamoDBStater

func NewDynamoDBStater(session *session.Session, backfillHrs int) (*DynamoDBStater, error)

func (*DynamoDBStater) ProcessedObjects

func (d *DynamoDBStater) ProcessedObjects() (map[string]time.Time, error)

list of processed objects

func (*DynamoDBStater) SetProcessed

func (d *DynamoDBStater) SetProcessed(s3object string) error

type FileStater

type FileStater struct {
	*sync.Mutex
	StateDir         string
	Service          string
	BackfillInterval time.Duration
}

FileStater is an implementation for indicating processing state using the local filesystem for backing storage.

func NewFileStater

func NewFileStater(stateDir, service string, backfillHrs int) *FileStater

func (*FileStater) ProcessedObjects

func (f *FileStater) ProcessedObjects() (map[string]time.Time, error)

func (*FileStater) SetProcessed

func (f *FileStater) SetProcessed(object string) error

type Record

type Record struct {
	S3Object string
	Time     time.Time
	TTL      int64 //future date formatted as unix seconds-since-epoch
}

Used for unmarshaling and adding objects to DynamoDB

type Stater

type Stater interface {
	// ProcessedObjects returns the full list of which objects have been
	// processed already.
	ProcessedObjects() (map[string]time.Time, error)

	// SetProcessed indicates that downloading, processing, and sending the
	// object to Honeycomb has been completed successfully.
	SetProcessed(object string) error
}

Stater lets us gain insight into the current state of object processing. It could be backed by the local filesystem, cloud abstractions such as DynamoDB, consistent value stores like etcd, etc.