nfdump

package module
v0.0.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 18, 2024 License: BSD-2-Clause Imports: 13 Imported by: 0

README

go-nfdump

Go Reference buildtest Go Report Card

This Go module allows to read and process files created by nfdump, the netflow/ipfix/sflow collector and processing tools.

This module is experimental and does not yet decode all available nfdump record extensions. It reads and processes only nfdump v2 files, which are created by nfdump-1.7.x. Files created with nfdump-1.6.x are recogized but skipped for decoding.

Expample to read and process a flow file:


package main

import (
	"flag"
	"fmt"
	"os"

	nfdump "github.com/phaag/go-nfdump"
)

var (
	fileName = flag.String("r", "", "nfdump file to read")
)

func main() {

	flag.CommandLine.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s [flags]\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()

	if len(*fileName) == 0 {
		fmt.Printf("Filename required\n")
		flag.PrintDefaults()
		os.Exit(255)
	}

	nffile := nfdump.New()

	if err := nffile.Open(*fileName); err != nil {
		fmt.Printf("Failed to open nf file: %v\n", err)
		os.Exit(255)
	}

	// print nffile stats
	fmt.Printf("nffile:\n%v", nffile)

	// Dump flow records
	recordChannel, _ := nffile.AllRecords()
	cnt := 0
	for record := range recordChannel {
		cnt++

		// check IP addresses in record for IPv4, or IPv6
		if record.IsIPv4() {
			fmt.Printf("Record %d is IPv4\n", cnt)
		} else if record.IsIPv6() {
			fmt.Printf("Record %d is IPv6\n", cnt)
		} else {
			fmt.Printf("Record %d has no IPs\n", cnt)
		}

		// sampling
 		packetInterval, spaceInterval := record.SamplerInfo(nffile)
		fmt.Printf("Record sampler info: packet interval: %d, space interval: %d\n",
               packetInterval, spaceInterval)
    
		// print the entire record using %v
		fmt.Printf("%v\n", record)

		// get generic extension and print ports
		// see nfxV3.go for all fields in genericFlow
		if genericFlow := record.GenericFlow(); genericFlow != nil {
			fmt.Printf("SrcPort: %d\n", genericFlow.SrcPort)
			fmt.Printf("DstPort: %d\n", genericFlow.DstPort)
		}

		// get src, dst ip address extension of record
		// can contain IPv4 or IPv6
		ipAddr := record.IP()
		if ipAddr != nil {
			// when printing as %v, Golang takes care about proper formating
			// as IPv4 or IPv6
			// see Golang standard library net.IP for more details to process IPs
			fmt.Printf("SrcIP: %v\n", ipAddr.SrcIP)
			fmt.Printf("DstIP: %v\n", ipAddr.DstIP)
		}
    
    if flowRecord.hasXlateIP {
      fmt.Sprintf("  SrcXlateIP  : %v\n", flowRecord.srcXlateIP)
      fmt.Sprintf("  DstXlateIP  : %v\n", flowRecord.dstXlateIP)
    }
    
    // get xlate ports and print
    if xlatePort := flowRecord.XlatePort(); xlatePort == nil {
      fmt.Printf("  Src X-Port  : %d\n", xlatePort.XlateSrcPort)
      fmt.Printf("  Dst X-Port  : %d\n", xlatePort.XlateDstPort)
    }
   
		/*
			// other extension
			// see nfxV3.go for all fields in the respectiv records
			// always check for nil return value as not every extension
			// is available
			flowMisc := record.FlowMisc()
			cntFlow := record.CntFlow()
			vLan := record.VLan()
			asRouting := record.AsRouting()
			bgpNextHop := record.BgpNextHop()
			ipNextHop := record.IpNextHop()
			
			// please note, sampling contains only references to exporter list
			// use record.SamplerInfo(nffile) to retrieve true sampling values
			sampling := record.Sampling()
		*/
	}
  
	// retrieve exporter list *after* all records are processed
	exporterList := nffile.GetExporterList()
	fmt.Printf("Exporter list:\n")
	for id, exporter := range exporterList {
		if exporter.IP != nil && id == int(exporter.SysId) { // valid exporter
			fmt.Printf("  SysID: %d, ID: %d, IP: %v, version: %d", 
                 exporter.SysId, exporter.Id, exporter.IP, exporter.Version)
			fmt.Printf(" Sequence failures: %d, packets: %d, flows: %d\n",
                 exporter.SequenceFailures, exporter.Packets, exporter.Flows)
		}
	}
}

The defs.go file includes nfdump's nfxV3.h header file to convert individual record extensions into appropriate Golang records. So far the generic, misc, flowCount, vlan and asRouting extensions as well as IPv4/IPv6 addresses are available through the interface. See the nfxV3.go file for its definitions.

If you modify the defs.go file, generate nfxV3.go use the go command

go generate ./...

All available extensions are visible in nfxV3.go.

Please note, that the interface may be subject to change, as this module is work in progress.

More element data blocks will follow, including the famous nfdump filter engine. Please submit your pull requests and/or bug reports via GitHub.

Documentation

Overview

Package nfdump provides an API for nfdump files

Package nfdump provides an API for nfdump files

Index

Constants

View Source 
const (
	NUM_FLAGS           = 4
	FLAG_NOT_COMPRESSED = 0x0  // records are not compressed
	FLAG_LZO_COMPRESSED = 0x1  // records are LZO compressed
	FLAG_ANONYMIZED     = 0x2  // flow data are anonimized
	FLAG_UNUSED         = 0x4  // unused
	FLAG_BZ2_COMPRESSED = 0x8  // records are BZ2 compressed
	FLAG_LZ4_COMPRESSED = 0x10 // records are LZ4 compressed
	COMPRESSION_MASK    = 0x19 // all compression bits
)
View Source 
const (
	EXnull            = uint(0x0)
	EXgenericFlowID   = uint16(0x1)
	EXipv4FlowID      = uint16(0x2)
	EXipv6FlowID      = uint16(0x3)
	EXflowMiscID      = uint16(0x4)
	EXcntFlowID       = uint16(0x5)
	EXvLanID          = uint16(0x6)
	EXasRoutingID     = uint16(0x7)
	EXbgpNextHopV4ID  = uint16(0x8)
	EXbgpNextHopV6ID  = uint16(0x9)
	EXipNextHopV4ID   = uint16(0xa)
	EXipNextHopV6ID   = uint16(0xb)
	EXipReceivedV4ID  = uint16(0xc)
	EXipReceivedV6ID  = uint16(0xd)
	EXsamplerInfoID   = uint16(0x12)
	EXinPayloadID     = uint16(0x1d)
	EXnselXlateIPv4ID = uint16(0x14)
	EXnselXlateIPv6ID = uint16(0x15)
	EXnselXlatePortID = uint16(0x16)
)
View Source 
const (
	V3_FLAG_EVENT   = uint(0x1)
	V3_FLAG_SAMPLED = uint(0x2)
	V3_FLAG_ANON    = uint(0x4)
)
View Source 
const (
	V3Record                = uint16(0xb)
	ExporterInfoRecordType  = uint16(0x7)
	ExporterStatRecordType  = uint16(0x8)
	SamplerLegacyRecordType = uint16(0x9)
	SamplerRecordType       = uint16(0xf)
)
View Source 
const BUFFSIZE = 5 * 1048576
View Source 
const BZ2_COMPRESSED = 2
View Source 
const LZ4_COMPRESSED = 3
View Source 
const LZO_COMPRESSED = 1
View Source 
const MAXEXTENSIONS = uint16(0x27)
View Source 
const MaxExporters = 256
View Source 
const NOT_COMPRESSED = 0
View Source 
const TYPE_IDENT = 0x8001
View Source 
const TYPE_STAT = 0x8002
View Source 
const ZSTD_COMPRESSED = 4

Variables

This section is empty.

Functions

This section is empty.

Types

type DataBlock 

type DataBlock struct {
	Header DataBlockHeader
	Data   []byte
}

type DataBlockHeader 

type DataBlockHeader struct {
	NumRecords uint32 // size of this block in bytes without this header
	Size       uint32 // size of this block in bytes without this header
	Type       uint16 // Block type
	// DATA_BLOCK_TYPE_3   3
	// DATA_BLOCK_TYPE_4   4
	Flags uint16
}

type EXXlateip added in v0.0.4 

type EXXlateip struct {
	SrcXIP net.IP
	DstXIP net.IP
}

type EXasRouting 

type EXasRouting struct {
	SrcAS uint32
	DstAS uint32
}

type EXbgpNextHop added in v0.0.3 

type EXbgpNextHop struct {
	IP net.IP
}

type EXcntFlow 

type EXcntFlow struct {
	Flows      uint64
	OutPackets uint64
	OutBytes   uint64
}

type EXflowMisc 

type EXflowMisc struct {
	Input         uint32
	Output        uint32
	SrcMask       uint8
	DstMask       uint8
	Dir           uint8
	DstTos        uint8
	BiFlowDir     uint8
	FlowEndReason uint8
	RevTcpFlags   uint8
	FragmentFlags uint8
}

type EXgenericFlow 

type EXgenericFlow struct {
	MsecFirst    uint64
	MsecLast     uint64
	MsecReceived uint64
	InPackets    uint64
	InBytes      uint64
	SrcPort      uint16
	DstPort      uint16
	Proto        uint8
	TcpFlags     uint8
	FwdStatus    uint8
	SrcTos       uint8
}

type EXinPayload added in v0.0.4 

type EXinPayload []byte

type EXip 

type EXip struct {
	SrcIP net.IP
	DstIP net.IP
}

type EXipNextHop added in v0.0.3 

type EXipNextHop struct {
	IP net.IP
}

type EXipReceived added in v0.0.3 

type EXipReceived struct {
	IP net.IP
}

type EXnselXlatePort added in v0.0.4 

type EXnselXlatePort struct {
	XlateSrcPort uint16
	XlateDstPort uint16
}

type EXsamplerInfo added in v0.0.4 

type EXsamplerInfo struct {
	SelectorID uint64
	Sysid      uint16
	Align      uint16
	Pad_cgo_0  [4]byte
}

type EXvLan 

type EXvLan struct {
	SrcVlan uint32
	DstVlan uint32
}

type Exporter added in v0.0.4 

type Exporter struct {
	IP net.IP // IP address

	SysId            uint16 // internal assigned ID
	Version          uint16 // netflow version
	Id               uint32 // exporter ID/Domain ID/Observation Domain ID assigned by the device
	Packets          uint64 // number of packets sent by this exporter
	Flows            uint64 // number of flow records sent by this exporter
	SequenceFailures uint32 // number of sequence failures
	SamplerList      []Sampler
	// contains filtered or unexported fields
}

type ExporterInfoRecord added in v0.0.4 

type ExporterInfoRecord struct {
	Type    uint16
	Size    uint16
	Version uint32
	Ip      [2]uint64
	Family  uint16
	Sysid   uint16
	Id      uint32
}

type FlowRecordV3 

type FlowRecordV3 struct {
	// contains filtered or unexported fields
}

func NewRecord 

func NewRecord(record []byte) (*FlowRecordV3, error)

Extract next flow record from []byte stream

func (*FlowRecordV3) AsRouting 

func (flowRecord *FlowRecordV3) AsRouting() *EXasRouting

Return asRouting extension

func (*FlowRecordV3) BgpNextHop added in v0.0.3 

func (flowRecord *FlowRecordV3) BgpNextHop() *EXbgpNextHop

Return bgp next hop IPv4 or IPv6

func (*FlowRecordV3) CntFlow 

func (flowRecord *FlowRecordV3) CntFlow() *EXcntFlow

Return out counter extension

func (*FlowRecordV3) FlowMisc 

func (flowRecord *FlowRecordV3) FlowMisc() *EXflowMisc

Return misc extension

func (*FlowRecordV3) GenericFlow 

func (flowRecord *FlowRecordV3) GenericFlow() *EXgenericFlow

Return generic extension

func (*FlowRecordV3) GetSamplerInfo added in v0.0.4 

func (flowRecord *FlowRecordV3) GetSamplerInfo(nfFile *NfFile)

func (*FlowRecordV3) IP 

func (flowRecord *FlowRecordV3) IP() *EXip

Return IP extension IPv4 or IPv6

func (*FlowRecordV3) IpNextHop added in v0.0.3 

func (flowRecord *FlowRecordV3) IpNextHop() *EXipNextHop

Return IP next hop IPv4 or IPv6

func (*FlowRecordV3) IpReceived added in v0.0.3 

func (flowRecord *FlowRecordV3) IpReceived() *EXipReceived

Return IP received IPv4 or IPv6

func (*FlowRecordV3) IsIPv4 added in v0.0.3 

func (flowRecord *FlowRecordV3) IsIPv4() bool

Return true, if record is a IPv4 flow

func (*FlowRecordV3) IsIPv6 added in v0.0.3 

func (flowRecord *FlowRecordV3) IsIPv6() bool

Return true, if record is a IPv4 flow

func (*FlowRecordV3) Payload added in v0.0.4 

func (flowRecord *FlowRecordV3) Payload() EXinPayload

Return payload

func (*FlowRecordV3) SamplerInfo added in v0.0.4 

func (flowRecord *FlowRecordV3) SamplerInfo(nfFile *NfFile) (int, int)

get sampler info for flow record

func (*FlowRecordV3) Sampling added in v0.0.4 

func (flowRecord *FlowRecordV3) Sampling() *EXsamplerInfo

Return bgp next hop IPv4 or IPv6

func (*FlowRecordV3) String 

func (flowRecord *FlowRecordV3) String() string

Return string for %v Printf()

func (*FlowRecordV3) VLan 

func (flowRecord *FlowRecordV3) VLan() *EXvLan

Return vlan extension

func (*FlowRecordV3) XlateIP added in v0.0.4 

func (flowRecord *FlowRecordV3) XlateIP() *EXXlateip

Return IP extension IPv4 or IPv6

func (*FlowRecordV3) XlatePort added in v0.0.4 

func (flowRecord *FlowRecordV3) XlatePort() *EXnselXlatePort

Return asRouting extension

type NfFile 

type NfFile struct {
	Header NfFileHeader

	StatRecord   StatRecord
	ExporterList []Exporter
	// contains filtered or unexported fields
}

func New 

func New() *NfFile

New returns a new empty NfFile object

func (*NfFile) AllRecords 

func (nfFile *NfFile) AllRecords() (chan *FlowRecordV3, error)

AllRecord takes an NfFile object and returns a channel of FlowRecordV3 it reads and uncompresses the data blocks with ReadDataBlocks Iterating over the channel reads all flow records

func (*NfFile) Close 

func (nfFile *NfFile) Close() error

Closes the current underlaying file

func (*NfFile) GetExporterList added in v0.0.4 

func (nfFile *NfFile) GetExporterList() []Exporter

Get exporter list

func (*NfFile) Ident 

func (nfFile *NfFile) Ident() string

Ident returns the identifier of the current NfFile object

func (*NfFile) Open 

func (nfFile *NfFile) Open(fileName string) error

Open opens an nffile given as string argument

func (*NfFile) ReadDataBlocks 

func (nfFile *NfFile) ReadDataBlocks() (chan DataBlock, error)

ReadDataBlocks iterates over the underlaying file and decompresses the data blocks A channel with all uncompressed data blocks is returned.

func (*NfFile) Stat 

func (nfFile *NfFile) Stat() StatRecord

Stat returns the stat record of the current NfFile object

func (*NfFile) String 

func (nfFile *NfFile) String() string

print %v string function if an NfFile object is printed String() is called

type NfFileHeader 

type NfFileHeader struct {
	Magic       uint16 // magic 0xA50C to recognize nfdump file type and endian type
	Version     uint16 // version of binary file layout. Valid: version 2
	NfVersion   uint32 // version of nfdump created this file
	Created     uint64 // file creat time
	Compression uint8  // type of compression
	// NOT_COMPRESSED 0
	// LZO_COMPRESSED 1
	// BZ2_COMPRESSED 2
	// LZ4_COMPRESSED 3
	// ZSTD_COMPRESSED 4
	Encryption uint8 // type of encryption
	// NOT_ENCRYPTED 0
	AppendixBlocks uint16 // number of blocks to read from appendix
	Unused         uint32 // unused. must be 0
	OffAppendix    uint64 // // offset in file for appendix blocks with additional data
	BlockSize      uint32 // max block size of a data block
	NumBlocks      uint32 // number of data blocks in file
}

type NfFileHeaderV1 

type NfFileHeaderV1 struct {
	Magic     uint16 // magic 0xA50C to recognize nfdump file type and endian type
	Version   uint16 // version of binary file layout. Valid: version 2
	Flags     uint32
	NumBlocks uint32    // number of data blocks in file
	Ident     [128]byte // string identifier for this file
}

type Sampler added in v0.0.4 

type Sampler struct {
	Id             int64
	Algorithm      uint16
	PacketInterval uint32
	SpaceInterval  uint32
}

type SamplerRecord added in v0.0.4 

type SamplerRecord struct {
	Type           uint16
	Size           uint16
	Sysid          uint16
	Algorithm      uint16
	Id             int64
	PacketInterval uint32
	SpaceInterval  uint32
}

type StatRecord 

type StatRecord struct {
	// overall stat
	Numflows   uint64
	Numbytes   uint64
	Numpackets uint64
	// flow stat
	NumflowsTcp   uint64
	NumflowsUdp   uint64
	NumflowsIcmp  uint64
	NumflowsOther uint64
	// bytes stat
	NumbytesTcp   uint64
	NumbytesUdp   uint64
	NumbytesIcmp  uint64
	NumbytesOther uint64
	// packet stat
	NumpacketsTcp   uint64
	NumpacketsUdp   uint64
	NumpacketsIcmp  uint64
	NumpacketsOther uint64
	// time window
	FirstSeen uint64
	LastSeen  uint64
	// other
	SequenceFailure uint64
}

Source Files

View all Source files

Directories

Path Synopsis
example
reader

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.