parser

Documentation

Index

• Constants
• func ParseFile(path string, wr *docwriter.DocWriter, rs *config.Resources, database string)
• type Conn
• type DNS
• type HTTP
• type PFile
• type Watcher
  • func NewWatcher(cfg *config.Resources, dbm *database.MetaDBHandle) *Watcher
  • func (w *Watcher) Run(dw *docwriter.DocWriter)

Constants

const (
	// BOOL reflects true or false, designated 'T' or 'F'
	BOOL = "bool"

	// COUNT is a numeric representation of a UINT_64 represented as either
	// a string of digits or a hex number. Note that hex numbers will begin
	// with the traditional 0x
	COUNT = "count"

	// INT is a numeric type representing an INT_64 represetned by a string
	// of digits preceded by either a '+' or a '-'. Note that INT may also
	// be expressed in hex and will maintian its leading sign ('-0xff')
	INT = "int"

	// DOUBLE is a numeric type representing a double-precision float.
	// Representation is a string of digits with an optional decimal point
	// as well as optional '+' or '-' proceding the number. The number may
	// also be optionally scaled with e notation. So 1234 123.4 -123.4
	// +1.234 and .003E-23 are examples of valid double types.
	DOUBLE = "double"

	// TIME is a temporal type representing an absolute time. Until found
	// otherwise it will be assumed that all time values are UNIX-NANO.
	TIME = "time"

	// INTERVAL is a temporal type representing relative time. An Interval
	// constant is represented by by a numeric constant followed by a time
	// unit which is one of usec, msec, sec, min, hr, or day. An 's' may
	// be appended to the unit so 3.5 min and 3.5mins represent the same
	// value. Finally an optional '-' negates an interval, denoting past
	// time. So -12 hr is read as "twelve hours in the past."
	INTERVAL = "interval"

	// STRING is a type used to hold character string values.
	STRING = "string"

	// PATTERN is a type used to represent regular expressions. Pattern
	// documentation can be found at
	// http://flex.sourceforge.net/manual/Patterns.html
	PATTERN = "pattern"

	// PORT is a type used to represent transport-level port numbers these
	// are typically represented as a number followed by one of /udp, /tcp,
	// /icmp, or /unkown.
	PORT = "port"

	// ADDR is a type used to represent an IP address. IPv4 addresses are
	// represented in dotted quad notation. IPv6 addresses are written in
	// colon hex notation as outlined in RFC 2373 (including the mixed
	// notation which allows dotted quad IPv4 addresses in the lower 32
	// bits) and further placed into brackets. So [::ffff:192.168.1.100]
	// can be used to represent the IPv4 address 192.168.1.100.
	ADDR = "addr"

	// SUBNET is a type used to represent a subnet in CIDR notation. So
	// 10.10.150.0/24 and [fe80::]/64 are valid subnets.
	SUBNET = "subnet"

	// ENUM is a type allowing the specification of a set of related
	// values that have no further structure.
	ENUM = "enum"

	// TABLE represents an associated array that maps from one set of
	// values to another. Values being mapped are refered to as indices and
	// the resulting map the yield.
	TABLE = "table"

	// SET is like table but the collection of indicies do not have to map
	// to any yield value.
	SET = "set"

	// VECTOR is a table which is always mapped by its count.
	VECTOR = "vector"

	// RECORD represents a collection of values each with a field name and
	// type.
	RECORD = "record"

	// FUNCTION represents a function type in bro script.
	FUNCTION = "function"

	// EVENT represents an event handler in bro script.
	EVENT = "event"

	// HOOK represents a bro script object best described as as the an
	// intersection of a function and an event.
	HOOK = "hook"

	// A file object which can be written to, but not read from (which is a
	// limitation of bro script and has nothing to do with brosync).
	FILE = "file"

	// OPAQUE represents data whos type is intentionally hidden, but whose
	// values may be passed to certain bro script builtins.
	OPAQUE = "opaque"

	// ANY is used to bypass strong typing in bro script.
	ANY = "any"
)

Further documentation on bros datatypes can be found on the bro website at: https://www.bro.org/sphinx/script-reference/types.html It is of value to note that many of theese types have applications specific to bro script and will likely never be implemented as types with any meaning in ai-hunt.

Functions

func ParseFile

func ParseFile(path string, wr *docwriter.DocWriter, rs *config.Resources, database string)

ParseFile generates a document parser and parses the file to the writer Pass this a started writer. Otherwise the writers will be started several times and may lock out unexpectedly.

Types

type Conn

type Conn struct {
	// ID is the id coming out of mongodb
	ID bson.ObjectId `bson:"_id,omitempty"`
	// TimeStamp of this connection
	TimeStamp int64 `bson:"ts" bro:"ts" brotype:"time"`
	// Uid is the Unique Id for this connection (generated by Bro)
	UID string `bson:"uid" bro:"uid" brotype:"string"`
	// Source is the source address for this connection
	Source string `bson:"id_origin_h" bro:"id.orig_h" brotype:"addr"`
	// SourcePort is the source port of this connection
	SourcePort int `bson:"id_origin_p" bro:"id.orig_p" brotype:"port"`
	// Destination is the destination of the connection
	Destination string `bson:"id_resp_h" bro:"id.resp_h" brotype:"addr"`
	// DestinationPort is the port at the destination host
	DestinationPort int `bson:"id_resp_p" bro:"id.resp_p" brotype:"port"`
	// Proto is the string protocol identifier for this connection
	Proto string `bson:"proto" bro:"proto" brotype:"enum"`
	// Service describes the service of this connection if there was one
	Service string `bson:"service" bro:"service" brotype:"string"`
	// Duration is the floating point representation of connection length
	Duration float64 `bson:"duration" bro:"duration" brotype:"interval"`
	// OrigBytes is the byte count coming from the origin
	OrigBytes int64 `bson:"orig_bytes" bro:"orig_bytes" brotype:"count"`
	// RespBytes is the byte count coming in on response
	RespBytes int64 `bson:"resp_bytes" bro:"resp_bytes" brotype:"count"`
	// ConnState has data describing the state of a connection
	ConnState string `bson:"conn_state" bro:"conn_state" brotype:"string"`
	// LocalOrigin denotes that the connection originated locally
	LocalOrigin bool `bson:"local_orig" bro:"local_orig" brotype:"bool"`
	// LocalResponse denote that the connection responded locally
	LocalResponse bool `bson:"local_resp" bro:"local_resp" brotype:"bool"`
	// MissedBytes keeps a count of bytes missed
	MissedBytes int64 `bson:"missed_bytes" bro:"missed_bytes" brotype:"count"`
	// History is a string containing historical information
	History string `bson:"history"  bro:"history" brotype:"string"`
	// OrigPkts is a count of origin packets
	OrigPkts int64 `bson:"orig_pkts"  bro:"orig_pkts" brotype:"count"`
	// OrigIpBytes is another origin data count
	OrigIPBytes int64 `bson:"orig_ip_bytes" bro:"orig_ip_bytes" brotype:"count"`
	// RespPkts counts response packets
	RespPkts int64 `bson:"resp_pkts" bro:"resp_pkts" brotype:"count"`
	// RespIpBytes gives the bytecount of response data
	RespIPBytes int64 `bson:"resp_ip_bytes" bro:"resp_ip_bytes" brotype:"count"`
	// TunnelParents lists tunnel parents
	TunnelParents string `bson:"tunnel_parents" bro:"tunnel_parents" brotype:"set[string]"`
}

Conn provides a data structure for bro's connection data

type DNS

type DNS struct {
	// ID contians the id set by mongodb
	ID bson.ObjectId `bson:"_id,omitempty"`
	// TimeStamp of this connection
	TimeStamp int64 `bson:"ts" bro:"ts" brotype:"time"`
	// Uid is the Unique Id for this connection (generated by Bro)
	UID string `bson:"uid" bro:"uid" brotype:"string"`
	// Source is the source address for this connection
	Source string `bson:"id_origin_h" bro:"id.orig_h" brotype:"addr"`
	// SourcePort is the source port of this connection
	SourcePort int `bson:"id_origin_p" bro:"id.orig_p" brotype:"port"`
	// Destination is the destination of the connection
	Destination string `bson:"id_resp_h" bro:"id.resp_h" brotype:"addr"`
	// DestinationPort is the port at the destination host
	DestinationPort int `bson:"id_resp_p" bro:"id.resp_p" brotype:"port"`
	// Proto is the string protocol identifier for this connection
	Proto string `bson:"proto" bro:"proto" brotype:"enum"`
	// TransID contains a 16 bit identifier assigned by the program that generated
	// the query
	TransID int64 `bson:"trans_id" bro:"trans_id" brotype:"count"`
	// RTT contains the round trip time of this request / response
	RTT float64 `bson:"RTT" bro:"rtt" brotype:"interval"`
	// Query contians the query string
	Query string `bson:"query" bro:"query" brotype:"string"`
	// QClass contains a the qclass of the query
	QClass int64 `bson:"qclass" bro:"qclass" brotype:"count"`
	// QClassName contains a descriptive name for the query
	QClassName string `bson:"qclass_name" bro:"qclass_name" brotype:"string"`
	// QType contains the value of the query type
	QType int64 `bson:"qtype" bro:"qtype" brotype:"count"`
	// QTypeName provides a descriptive name for the query
	QTypeName string `bson:"qtype_name" bro:"qtype_name" brotype:"string"`
	// RCode contains the response code value from the DNS messages
	RCode int64 `bson:"rcode" bro:"rcode" brotype:"count"`
	// RCodeName provides a descriptive name for RCode
	RCodeName string `bson:"rcode_name" bro:"rcode_name" brotype:"string"`
	// AA represents the state of the authoritive answer bit of the resp messages
	AA bool `bson:"AA" bro:"AA" brotype:"bool"`
	// TC represents the truncation bit of the message
	TC bool `bson:"TC" bro:"TC" brotype:"bool"`
	// RD represens the recursion desired bit of the message
	RD bool `bson:"RD" bro:"RD" brotype:"bool"`
	// RA represents the recursion available bit of the message
	RA bool `bson:"RA" bro:"RA" brotype:"bool"`
	// Z represents the state of a reseverd field that should be zero in qll queries
	Z int64 `bson:"Z" bro:"Z" brotype:"count"`
	// Answers contains the set of resource descriptions in the query answer
	Answers string `bson:"answers" bro:"answers" brotype:"vector[string]"`
	// TTLs contians a vector of interval type time to live values
	TTLs string `bson:"TTLs" bro:"TTLs" brotype:"vector[interval]"`
	// Rejected indicates if this query was rejected or not
	Rejected bool `bson:"rejected" bro:"rejected" brotype:"bool"`
}

DNS provides a data structure for entries in the bro DNS log

type HTTP

type HTTP struct {
	// ID is the object id as set by mongodb
	ID bson.ObjectId `bson:"_id,omitempty"`
	// TimeStamp of this connection
	TimeStamp int64 `bson:"ts" bro:"ts" brotype:"time"`
	// Uid is the Unique Id for this connection (generated by Bro)
	UID string `bson:"uid" bro:"uid" brotype:"string"`
	// Source is the source address for this connection
	Source string `bson:"id_origin_h" bro:"id.orig_h" brotype:"addr"`
	// SourcePort is the source port of this connection
	SourcePort int `bson:"id_origin_p" bro:"id.orig_p" brotype:"port"`
	// Destination is the destination of the connection
	Destination string `bson:"id_resp_h" bro:"id.resp_h" brotype:"addr"`
	// DestinationPort is the port at the destination host
	DestinationPort int `bson:"id_resp_p" bro:"id.resp_p" brotype:"port"`
	// Transdepth is the ordinal value of requests into a pipeline transaction
	TransDepth int64 `bson:"trans_depth" bro:"trans_depth" brotype:"count"`
	// Version is the value of version in the request
	Version string `bson:"version" bro:"version" brotype:"string"`
	// Method is the request method used
	Method string `bson:"method" bro:"method" brotype:"string"`
	// Host is the value of the HOST header
	Host string `bson:"host" bro:"host" brotype:"string"`
	// URI is the uri used in this request
	URI string `bson:"uri" bro:"uri" brotype:"string"`
	// Referrer is the value of the referrer header in the request
	Referrer string `bson:"referrer" bro:"referrer" brotype:"string"`
	// UserAgent gives the user agent from the request
	UserAgent string `bson:"user_agent" bro:"user_agent" brotype:"string"`
	// ReqLen holds the length of the request body uncompressed
	ReqLen int64 `bson:"request_body_len" bro:"request_body_len" brotype:"count"`
	// RespLen hodls the length of the response body uncompressed
	RespLen int64 `bson:"response_body_len" bro:"response_body_len" brotype:"count"`
	// StatusCode holds the status result
	StatusCode int64 `bson:"status_code" bro:"status_code" brotype:"count"`
	// StatusMsg contains a string status message returned by the server
	StatusMsg string `bson:"status_msg" bro:"status_msg" brotype:"string"`
	// InfoCode holds the last seen 1xx informational reply code
	InfoCode int64 `bson:"info_code" bro:"info_code" brotype:"count"`
	// InfoMsg holds the last seen 1xx message string
	InfoMsg string `bson:"info_msg" bro:"info_msg" brotype:"string"`
	// FileName contains the name of the requested file
	FileName string `bson:"filename" bro:"filename" brotype:"string"`
	// Tags contains a set of indicators of various attributes related to a particular req and
	// response pair
	Tags string `bson:"tags" bro:"tags" brotype:"set[enum]"`
	// UserName will contain a username in the case of basic auth implementation
	UserName string `bson:"username" bro:"username" brotype:"string"`
	// Password will contain a password in the case of basic auth implementation
	Password string `bson:"password" bro:"password" brotype:"string"`
	// Proxied contains all headers that indicate a request was proxied
	Proxied string `bson:"proxied" bro:"proxied" brotype:"set[string]"`
	// OrigFuids contains an ordered vector of uniq file IDs
	OrigFuids string `bson:"orig_fuids" bro:"orig_fuids" brotype:"vector[string]"`
	// OrigFilenames contains an ordered vector of filenames from the client
	OrigFilenames string `bson:"orig_filenames" bro:"orig_filenames" brotype:"vector[string]"`
	// OrigMimeTypes contains an ordered vector of mimetypes
	OrigMimeTypes string `bson:"orig_mime_types" bro:"orig_mime_types" brotype:"vector[string]"`
	// RespFuids contains an ordered vector of unique file IDs in the response
	RespFuids string `bson:"resp_fuids" bro:"resp_fuids" brotype:"vector[string]"`
	// RespFilenames contains an ordered vector of unique files in the response
	RespFilenames string `bson:"resp_filenames" bro:"resp_filenames" brotype:"vector[string]"`
	// RespMimeTypes contains an ordered vector of unique MIME entities in the HTTP response body
	RespMimeTypes string `bson:"resp_mime_types" bro:"resp_mime_types" brotype:"vector[string]"`
}

HTTP provides a data structure for entries in bro's HTTP log file

type PFile

type PFile struct {
	ID       bson.ObjectId `bson:"_id,omitempty"`
	Path     string        `bson:"filepath"`
	Hash     string        `bson:"hash"`
	Length   int64         `bson:"length"`
	Parsed   int64         `bson:"time_complete"`
	Mod      time.Time     `bson:"modified"`
	DataBase string        `bson:"database"`
}

PFile retains everything we need to know about a given file

type Watcher

type Watcher struct {
	Meta *database.MetaDBHandle // Handle to the metadata
	// contains filtered or unexported fields
}

Watcher provides an interface to keep up with a directory

func NewWatcher

func NewWatcher(cfg *config.Resources, dbm *database.MetaDBHandle) *Watcher

NewWatcher takes a top level directory to watch and returns a watcher

func (*Watcher) Run

func (w *Watcher) Run(dw *docwriter.DocWriter)

Run simply runs the subcomponents in order building out the database