inetdata

README

Internet Data Processing Tools

Process internet data from various sources. Works with inetdata

Dependencies

Ubuntu 16.04

$ sudo apt-get install build-essential git make pigz p7zip-full libmtbl-dev mtbl-bin pkg-config

Golang

• Download the latest golang binary (1.8+) from https://golang.org/dl/
• Extract to the filesystem with: # tar -C /usr/local -xzf go$VERSION.$OS-$ARCH.tar.gz

Build

Configure your GO environment if you haven't done so, by adding the following to ~/.bashrc

$ echo 'export GOPATH=$HOME' >> ~/.bashrc
$ echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> ~/.bashrc

$ source ~/.bashrc

Clone this repository into the correct path:

$ mkdir -p $GOPATH/src/github.com/hdm/
$ cd $GOPATH/src/github.com/hdm/
$ git clone https://github.com/hdm/inetdata-parsers.git

Install

$ cd $GOPATH/src/github.com/hdm/inetdata-parsers/
$ make

Number of parallel builds: 3
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/mapi
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/mq
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-arin-xml2json
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-csvsplit
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-arin-org2cidrs
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-csv2mtbl
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-dns2mtbl
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-ct2csv
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-ct2hostnames
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-ct2hostnames-sync
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-ct2mtbl
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-csvrollup
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-lines2mtbl
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-hostnames2domains
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-json2mtbl
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-sonardnsv2-split
-->     linux/amd64: github.com/fathom6/inetdata-parsers/cmd/inetdata-zone2csv

Documentation

Index

• Constants
• Variables
• func AddressesFromCIDR(cidr string, o chan<- string)
• func IPv42UInt(ips string) (uint32, error)
• func IPv4Range2CIDRs(sIP string, eIP string) ([]string, error)
• func IPv4UIntRange2CIDRs(sI uint32, eI uint32) []string
• func PrintVersion(app string)
• func ReadLines(input *os.File, out chan<- string) error
• func ReadLinesFromReader(input io.Reader, out chan<- string) error
• func ReverseKey(s string) string
• func ReverseKeyBytes(s []byte) []byte
• func UInt2IPv4(ipi uint32) string

Constants

const MTBL_KEY_LIMIT = 1024 * 1024 * 512

const MTBL_VAL_LIMIT = 1024 * 1024 * 1024

Variables

var IPv4MaskSizes = []uint32{
	2147483648,
	1073741824,
	536870912,
	268435456,
	134217728,
	67108864,
	33554432,
	16777216,
	8388608,
	4194304,
	2097152,
	1048576,
	524288,
	262144,
	131072,
	65536,
	32768,
	16384,
	8192,
	4096,
	2048,
	1024,
	512,
	256,
	128,
	64,
	32,
	16,
	8,
	4,
	2,
	1,
}

IPv4MaskSizes is a precalculated lookup table for IPv4 CIDR mask sizes

var IPv4Masks = map[uint32]uint32{
	1:          32,
	2:          31,
	4:          30,
	8:          29,
	16:         28,
	32:         27,
	64:         26,
	128:        25,
	256:        24,
	512:        23,
	1024:       22,
	2048:       21,
	4096:       20,
	8192:       19,
	16384:      18,
	32768:      17,
	65536:      16,
	131072:     15,
	262144:     14,
	524288:     13,
	1048576:    12,
	2097152:    11,
	4194304:    10,
	8388608:    9,
	16777216:   8,
	33554432:   7,
	67108864:   6,
	134217728:  5,
	268435456:  4,
	536870912:  3,
	1073741824: 2,
	2147483648: 1,
}

IPv4Masks is a precalculated lookup table for IPv4 CIDR

var MTBLCompressionTypes = map[string]int{
	"none":   mtbl.COMPRESSION_NONE,
	"snappy": mtbl.COMPRESSION_SNAPPY,
	"zlib":   mtbl.COMPRESSION_ZLIB,
	"lz4":    mtbl.COMPRESSION_LZ4,
	"lz4hc":  mtbl.COMPRESSION_LZ4HC,
}

var MatchIPv4 = regexp.MustCompile(`^(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))$`)

MatchIPv4 is a regular expression for validating IPv4 addresses

var MatchIPv6 = regexp.MustCompile(`^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?$`)

MatchIPv6 is a regular expression for validating IPv6 addresses

var Match_SHA1 = regexp.MustCompile(`^[a-zA-Z0-9]{40}$`)

var Split_WS = regexp.MustCompile(`\s+`)

var Version string = "0.0.37"

Functions

func AddressesFromCIDR

func AddressesFromCIDR(cidr string, o chan<- string)

AddressesFromCIDR parses a CIDR and writes individual IPs to a channel

func IPv42UInt

func IPv42UInt(ips string) (uint32, error)

IPv42UInt converts IPv4 addresses to unsigned integers

func IPv4Range2CIDRs

func IPv4Range2CIDRs(sIP string, eIP string) ([]string, error)

IPv4Range2CIDRs converts a start and stop IPv4 range to a list of CIDRs

func IPv4UIntRange2CIDRs

func IPv4UIntRange2CIDRs(sI uint32, eI uint32) []string

IPv4UIntRange2CIDRs converts a range of insigned integers into IPv4 CIDRs

func PrintVersion

func PrintVersion(app string)

func ReadLines

func ReadLines(input *os.File, out chan<- string) error

func ReadLinesFromReader

func ReadLinesFromReader(input io.Reader, out chan<- string) error

func ReverseKey

func ReverseKey(s string) string

func ReverseKeyBytes

func ReverseKeyBytes(s []byte) []byte

func UInt2IPv4

func UInt2IPv4(ipi uint32) string

UInt2IPv4 converts unsigned integers to IPv4 addresses