nfdump

package module
v0.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 7, 2023 License: BSD-2-Clause Imports: 11 Imported by: 0

README

go-nfdump

Go Reference buildtest Go Report Card

This Go module allows to read and process files created by nfdump, the netflow/ipfix/sflow collector and processing tools.

This module is experimental and does not yet decode all available nfdump record extensions. It reads and processes only nfdump v2 files, which are created by nfdump-1.7.x. Files created with nfdump-1.6.x are recogized but skipped for decoding.

Expample to read and process a flow file:


package main

import (
	"flag"
	"fmt"
	"os"

	nfdump "github.com/phaag/go-nfdump"
)

var (
	fileName = flag.String("r", "", "nfdump file to read")
)

func main() {

	flag.CommandLine.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s [flags]\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.Parse()

	if len(*fileName) == 0 {
		fmt.Printf("Filename required\n")
		flag.PrintDefaults()
		os.Exit(255)
	}

	nffile := nfdump.New()

	if err := nffile.Open(*fileName); err != nil {
		fmt.Printf("Failed to open nf file: %v\n", err)
		os.Exit(255)
	}

	// print nffile stats
	fmt.Printf("nffile:\n%v", nffile)

	// Dump flow records
	recordChannel, _ := nffile.AllRecords()
	cnt := 0
	for record := range recordChannel {
		cnt++
		fmt.Printf("record: %d\n%v\n", cnt, record)
		genericFlow := record.GenericFlow()
		if genericFlow != nil {
			fmt.Printf("SrcPort: %d\n", genericFlow.SrcPort)
			fmt.Printf("DstPort: %d\n", genericFlow.DstPort)
		}
		ipAddr := record.IP()
		if ipAddr != nil {
			fmt.Printf("SrcIP: %v\n", ipAddr.SrcIP)
			fmt.Printf("DstIP: %v\n", ipAddr.DstIP)
		}
		/*
			other extension
			flowMisc := record.FlowMisc()
			cntFlow := record.CntFlow()
			vLan := record.VLan()
			asRouting := record.AsRouting()
		*/
	}
}

The defs.go file includes nfdump's nfxV3.h header file to convert individual record extensions into appropriate Golang records. So far the generic, misc, flowCount, vlan and asRouting extensions as well as IPv4/IPv6 addresses are available through the interface. See the nfxV3.go file for its definitions.

If you modify the defs.go file, generate nfxV3.go use the go command

go generate ./...

Please note, that the interface may be subject to change, as this module is work in progress.

More element data blocks will follow, including the famous nfdump filter engine.

Documentation

Overview

Package nfdump provides an API for nfdump files

Index

Constants

View Source 
const (
	NUM_FLAGS           = 4
	FLAG_NOT_COMPRESSED = 0x0  // records are not compressed
	FLAG_LZO_COMPRESSED = 0x1  // records are LZO compressed
	FLAG_ANONYMIZED     = 0x2  // flow data are anonimized
	FLAG_UNUSED         = 0x4  // unused
	FLAG_BZ2_COMPRESSED = 0x8  // records are BZ2 compressed
	FLAG_LZ4_COMPRESSED = 0x10 // records are LZ4 compressed
	COMPRESSION_MASK    = 0x19 // all compression bits
)
View Source 
const BZ2_COMPRESSED = 2
View Source 
const EXasRoutingID = uint16(0x7)
View Source 
const EXcntFlowID = uint16(0x5)
View Source 
const EXflowMiscID = uint16(0x4)
View Source 
const EXgenericFlowID = uint16(0x1)
View Source 
const EXipv4FlowID = uint16(0x2)
View Source 
const EXipv6FlowID = uint16(0x3)
View Source 
const EXnull = uint(0x0)
View Source 
const EXvLanID = uint16(0x6)
View Source 
const LZ4_COMPRESSED = 3
View Source 
const LZO_COMPRESSED = 1
View Source 
const MAXEXTENSIONS = uint16(0x26)
View Source 
const NOT_COMPRESSED = 0
View Source 
const TYPE_IDENT = 0x8001
View Source 
const TYPE_STAT = 0x8002
View Source 
const V3Record = uint16(0xb)
View Source 
const V3_FLAG_ANON = uint(0x4)
View Source 
const V3_FLAG_EVENT = uint(0x1)
View Source 
const V3_FLAG_SAMPLED = uint(0x2)

Variables

This section is empty.

Functions

This section is empty.

Types

type DataBlock 

type DataBlock struct {
	Header DataBlockHeader
	Data   []byte
}

type DataBlockHeader 

type DataBlockHeader struct {
	NumRecords uint32 // size of this block in bytes without this header
	Size       uint32 // size of this block in bytes without this header
	Type       uint16 // Block type
	// DATA_BLOCK_TYPE_3   3
	// DATA_BLOCK_TYPE_4   4
	Flags uint16
}

type EXasRouting 

type EXasRouting struct {
	SrcAS uint32
	DstAS uint32
}

type EXcntFlow 

type EXcntFlow struct {
	Flows      uint64
	OutPackets uint64
	OutBytes   uint64
}

type EXflowMisc 

type EXflowMisc struct {
	Input         uint32
	Output        uint32
	SrcMask       uint8
	DstMask       uint8
	Dir           uint8
	DstTos        uint8
	BiFlowDir     uint8
	FlowEndReason uint8
	RevTcpFlags   uint8
	Fill          uint8
}

type EXgenericFlow 

type EXgenericFlow struct {
	MsecFirst    uint64
	MsecLast     uint64
	MsecReceived uint64
	InPackets    uint64
	InBytes      uint64
	SrcPort      uint16
	DstPort      uint16
	Proto        uint8
	TcpFlags     uint8
	FwdStatus    uint8
	SrcTos       uint8
}

type EXip 

type EXip struct {
	SrcIP net.IP
	DstIP net.IP
}

type EXvLan 

type EXvLan struct {
	SrcVlan uint32
	DstVlan uint32
}

type FlowRecordV3 

type FlowRecordV3 struct {
	// contains filtered or unexported fields
}

func NewRecord 

func NewRecord(record []byte) *FlowRecordV3

Extract next flow record from []byte stream

func (*FlowRecordV3) AsRouting 

func (flowRecord *FlowRecordV3) AsRouting() *EXasRouting

Return asRouting extension

func (*FlowRecordV3) CntFlow 

func (flowRecord *FlowRecordV3) CntFlow() *EXcntFlow

Return out counter extension

func (*FlowRecordV3) FlowMisc 

func (flowRecord *FlowRecordV3) FlowMisc() *EXflowMisc

Return misc extension

func (*FlowRecordV3) GenericFlow 

func (flowRecord *FlowRecordV3) GenericFlow() *EXgenericFlow

Return generic extension

func (*FlowRecordV3) IP 

func (flowRecord *FlowRecordV3) IP() *EXip

Return IP extension IPv4 or IPv6

func (*FlowRecordV3) String 

func (flowRecord *FlowRecordV3) String() string

Return string for %v Printf()

func (*FlowRecordV3) VLan 

func (flowRecord *FlowRecordV3) VLan() *EXvLan

Return vlan extension

type NfFile 

type NfFile struct {
	Header NfFileHeader

	StatRecord StatRecord
	// contains filtered or unexported fields
}

func New 

func New() *NfFile

New returns a new empty NfFile object

func (*NfFile) AllRecords 

func (nfFile *NfFile) AllRecords() (chan *FlowRecordV3, error)

AllRecord takes an NfFile object and returns a channel of FlowRecordV3 it reads and uncompresses the data blocks with ReadDataBlocks Iterating over the channel reads all flow records

func (*NfFile) Close 

func (nfFile *NfFile) Close() error

Closes the current underlaying file

func (*NfFile) Ident 

func (nfFile *NfFile) Ident() string

Ident returns the identifier of the current NfFile object

func (*NfFile) Open 

func (nfFile *NfFile) Open(fileName string) error

Open opens an nffile given as string argument

func (*NfFile) ReadDataBlocks 

func (nfFile *NfFile) ReadDataBlocks() (chan DataBlock, error)

ReadDataBlocks iterates over the underlaying file and decompresses the data blocks A channel with all uncompressed data blocks is returned.

func (*NfFile) Stat 

func (nfFile *NfFile) Stat() StatRecord

Stat returns the stat record of the current NfFile object

func (*NfFile) String 

func (nfFile *NfFile) String() string

print %v string function if an NfFile object is printed String() is called

type NfFileHeader 

type NfFileHeader struct {
	Magic       uint16 // magic 0xA50C to recognize nfdump file type and endian type
	Version     uint16 // version of binary file layout. Valid: version 2
	NfVersion   uint32 // version of nfdump created this file
	Created     uint64 // file creat time
	Compression uint8  // type of compression
	// NOT_COMPRESSED 0
	// LZO_COMPRESSED 1
	// BZ2_COMPRESSED 2
	// LZ4_COMPRESSED 3
	Encryption uint8 // type of encryption
	// NOT_ENCRYPTED 0
	AppendixBlocks uint16 // number of blocks to read from appendix
	Unused         uint32 // unused. must be 0
	OffAppendix    uint64 // // offset in file for appendix blocks with additional data
	BlockSize      uint32 // max block size of a data block
	NumBlocks      uint32 // number of data blocks in file
}

type NfFileHeaderV1 

type NfFileHeaderV1 struct {
	Magic     uint16 // magic 0xA50C to recognize nfdump file type and endian type
	Version   uint16 // version of binary file layout. Valid: version 2
	Flags     uint32
	NumBlocks uint32    // number of data blocks in file
	Ident     [128]byte // string identifier for this file
}

type StatRecord 

type StatRecord struct {
	// overall stat
	Numflows   uint64
	Numbytes   uint64
	Numpackets uint64
	// flow stat
	NumflowsTcp   uint64
	NumflowsUdp   uint64
	NumflowsIcmp  uint64
	NumflowsOther uint64
	// bytes stat
	NumbytesTcp   uint64
	NumbytesUdp   uint64
	NumbytesIcmp  uint64
	NumbytesOther uint64
	// packet stat
	NumpacketsTcp   uint64
	NumpacketsUdp   uint64
	NumpacketsIcmp  uint64
	NumpacketsOther uint64
	// time window
	FirstSeen uint64
	LastSeen  uint64
	// other
	SequenceFailure uint64
}

Source Files

View all Source files

Directories

Path Synopsis
example
reader command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.