data

Documentation

Index

• type FileToProcessInfo
  • func MakeFileToProcessInfo(bucket string, key string, size int64, wg *sync.WaitGroup) *FileToProcessInfo
  • func (f *FileToProcessInfo) Done()
• type LogEntry
  • func ParseLogEntry(line *string) (*LogEntry, error)
• type LogToProcess
  • func MakeLogToProcess(log *LogEntry, file *FileToProcessInfo) *LogToProcess
  • func (f *LogToProcess) Combine(record *LogToProcess)
  • func (f *LogToProcess) Done()

Types

type FileToProcessInfo

type FileToProcessInfo struct {
	Bucket       string `json:"bucket"`
	Key          string `json:"key"`
	Size         int64  `json:"size"`
	TimestampVal int64  `json:"timestamp"`

	UncompressedSize int64     `json:"-"`
	Timestamp        time.Time `json:"-"`
	// contains filtered or unexported fields
}

FileToProcessInfo describes an S3 file to process and the waitgroup to notify when done!

func MakeFileToProcessInfo

func MakeFileToProcessInfo(bucket string, key string, size int64, wg *sync.WaitGroup) *FileToProcessInfo

MakeFileToProcessInfo builds a new FileToProcessInfo object and returns it.

func (*FileToProcessInfo) Done

func (f *FileToProcessInfo) Done()

Done marks this file as processed.

type LogEntry

type LogEntry struct {
	/*
		flowID is an identifier for a FLOW.
		It is used when collecting multiple log records for the same flow together
		Built as: srcaddr + ":" + srcport + "-" + dstaddr + ":" + dstport + "-" + InterfaceID + "-" + action
	*/
	FlowID        string // Ignored for Parquet...
	Version       int32  // Ignore for Parquet - we only support version 2 right now
	AccountID     string `parquet:"name=account, type=UTF8, encoding=PLAIN_DICTIONARY"`
	InterfaceID   string `parquet:"name=interfaceId, type=UTF8, encoding=PLAIN_DICTIONARY"`
	SrcAddress    string `parquet:"name=srcAddress, type=UTF8, encoding=PLAIN_DICTIONARY"`
	DstAddress    string `parquet:"name=dstAddress, type=UTF8, encoding=PLAIN_DICTIONARY"`
	SrcPort       int32  `parquet:"name=srcPort, type=INT32"`
	DstPort       int32  `parquet:"name=dstPort, type=INT32"`
	Protocol      int32  `parquet:"name=protocol, type=INT32"`
	ProtocolName  string //`parquet:"name=protocolName, type=UTF8, encoding=PLAIN_DICTIONARY"`
	Packets       int64  `parquet:"name=packets, type=INT64"`
	Bytes         int64  `parquet:"name=bytes, type=INT64"`
	Start         time.Time
	StartVal      int64 `parquet:"name=start, type=TIMESTAMP_MILLIS"`
	End           time.Time
	EndVal        int64  `parquet:"name=end, type=TIMESTAMP_MILLIS"`
	Action        bool   `parquet:"name=action, type=BOOLEAN"`
	LogStatus     int32  // `parquet:"name=logStatus, type=INT32"` // 0 = OK, 1 = NODATA, 2 = SKIP
	NumLogEntries int32  `parquet:"name=ofRawlogEntries, type=INT32"`
	OriginHash    uint32 // `parquet:"name=originHash, type=UINT_32"`
}

LogEntry describes a single line of log data

func ParseLogEntry

func ParseLogEntry(line *string) (*LogEntry, error)

ParseLogEntry parses a raw FlowLog Entry line and returns a LogEntry.FileToProcessInfo

type LogToProcess

type LogToProcess struct {
	Log *LogEntry
	// contains filtered or unexported fields
}

LogToProcess holds a log entry and a reference to the associated wait group that needs to be makred complete when done.

func MakeLogToProcess

func MakeLogToProcess(log *LogEntry, file *FileToProcessInfo) *LogToProcess

MakeLogToProcess creates a new LogToProcess and returns it.

func (*LogToProcess) Combine

func (f *LogToProcess) Combine(record *LogToProcess)

Combine will merge to LogToProcess structs together

func (*LogToProcess) Done

func (f *LogToProcess) Done()

Mark this LogEntry as processed.