netident

README

netident

Identify network providers, search engine bots, and cloud platforms by matching IP, PTR, WHOIS, and ASN data against a weighted JSON rule database.

Install

Library:

go get github.com/aredoff/netident

CLI — download a binary from GitHub Releases.

CLI

Identify a provider by IP, PTR, WHOIS, or ASN data:

netident -ip 66.249.79.1 -ptr crawl-66-249-79-1.googlebot.com

Output:

provider=googlebot name="Google Bot" category=bot score=1000
  ptr "*.googlebot.com" weight=1000

More examples:

netident -ip 8.8.8.8 -netname GOOGLE-CLOUD
netident -asn 15169 -asn-name GOOGLE
netident -netmail abuse@amazonaws.com -ptr ec2-1-2-3-4.compute.amazonaws.com
netident -version
netident -validate ./providers.json

Validate a custom config:

netident -validate ./my-providers.json

Output:

config ok: ./my-providers.json

Flags:

Flag	Description
-ip	IP address
-ptr	PTR record
-netname	WHOIS netname
-netmail	WHOIS netmail
-asn	ASN number
-asn-name	ASN organization name
-asn-mail	ASN abuse email
-validate	Validate providers.json, load config, and run a probe identify
-config	Path to custom providers.json (default: embedded production config)
-cache-dir	Directory for network URL cache
-version	Print version and exit

At least one input flag is required for identify mode. -validate runs standalone and does not require input flags.

Library usage

package main

import (
    "context"
    "net"

    "github.com/aredoff/netident"
)

func main() {
    det, err := netident.New()
    if err != nil {
        panic(err)
    }

    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    if err := det.Start(ctx); err != nil {
        panic(err)
    }
    defer det.Close()

    result := det.Identify(netident.Input{
        IP:  net.ParseIP("66.249.79.1"),
        PTR: "crawl-66-249-79-1.googlebot.com",
    })

    if result.OK && result.Category == netident.CategoryBot {
        // never block bots
        return
    }
}

Lifecycle

Method	Description
New*	Load JSON config, compile rules, read cache. No goroutines.
Start(ctx)	Start background network URL updater.
Close()	Stop updater and wait for shutdown.
Identify	Match input against all providers, return best result.

Identify works immediately after New using static rules and cached CIDR lists.

Config format

Each provider has weighted rules:

{
  "version": 1,
  "defaults": {"min_score": 1, "rule_weight": 10},
  "providers": [
    {
      "id": "googlebot",
      "name": "Google Bot",
      "category": "bot",
      "rules": {
        "ptr": [{"match": "*.googlebot.com", "weight": 1000}],
        "network_urls": [{
          "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
          "format": "google_prefixes",
          "weight": 900
        }]
      }
    }
  ]
}

All rules use {"match": "...", "weight": N}. Omitted weight defaults to defaults.rule_weight.

Supported rule fields: ptr, networks, network_urls, netname, netmail, asn, asn_name, asn_mail.

Categories: bot, cloud, cdn, hosting, isp, other.

License

MIT

Documentation

Overview

Package netident identifies network providers, bots, and cloud hosts by matching IP, PTR, WHOIS, and ASN fields against a weighted JSON rule database.

Index

• type Category
• type Detector
  • func New(opts ...Option) (*Detector, error)
  • func NewFromFile(path string, opts ...Option) (*Detector, error)
  • func NewFromJSON(r io.Reader, opts ...Option) (*Detector, error)
  • func (d *Detector) Close() error
  • func (d *Detector) Identify(input Input) Result
  • func (d *Detector) IdentifyAll(input Input) []Result
  • func (d *Detector) Ready(ctx context.Context) error
  • func (d *Detector) Start(ctx context.Context) error
• type FileCache
  • func NewFileCache(dir string) *FileCache
  • func (c *FileCache) Load(key string) ([]byte, time.Time, error)
  • func (c *FileCache) Store(key string, data []byte) error
• type Input
• type Match
• type NetworkCache
• type Option
  • func WithCache(c NetworkCache) Option
  • func WithCacheDir(dir string) Option
  • func WithHTTPClient(c *http.Client) Option
  • func WithLogger(l *slog.Logger) Option
• type Result

Examples

• Detector.IdentifyAll
• New (Identify)

Types

type Category

type Category string

const (
	CategoryBot     Category = "bot"
	CategoryCloud   Category = "cloud"
	CategoryCDN     Category = "cdn"
	CategoryHosting Category = "hosting"
	CategoryISP     Category = "isp"
	CategoryOther   Category = "other"
)

type Detector

type Detector struct {
	// contains filtered or unexported fields
}

func New

func New(opts ...Option) (*Detector, error)

Example (Identify)

package main

import (
	"fmt"

	"github.com/aredoff/netident"
)

func main() {
	d, err := netident.New()
	if err != nil {
		fmt.Println(err)
		return
	}

	result := d.Identify(netident.Input{
		PTR: "crawl-66-249-79-1.googlebot.com",
	})

	fmt.Println(result.ProviderID, result.Score, result.OK)
}

Output:
googlebot 1000 true

func NewFromFile

func NewFromFile(path string, opts ...Option) (*Detector, error)

func NewFromJSON

func NewFromJSON(r io.Reader, opts ...Option) (*Detector, error)

func (*Detector) Close

func (d *Detector) Close() error

func (*Detector) Identify

func (d *Detector) Identify(input Input) Result

func (*Detector) IdentifyAll

func (d *Detector) IdentifyAll(input Input) []Result

Example

package main

import (
	"fmt"
	"net"

	"github.com/aredoff/netident"
)

func main() {
	d, err := netident.NewFromFile("testdata/providers_minimal.json")
	if err != nil {
		fmt.Println(err)
		return
	}

	ip := net.ParseIP("66.249.79.1")
	all := d.IdentifyAll(netident.Input{IP: ip})
	if len(all) == 0 {
		fmt.Println("none")
		return
	}
	fmt.Println(all[0].ProviderID, all[0].Score)
}

Output:
googlebot 900

func (*Detector) Ready

func (d *Detector) Ready(ctx context.Context) error

func (*Detector) Start

func (d *Detector) Start(ctx context.Context) error

type FileCache

type FileCache struct {
	// contains filtered or unexported fields
}

func NewFileCache

func NewFileCache(dir string) *FileCache

func (*FileCache) Load

func (c *FileCache) Load(key string) ([]byte, time.Time, error)

func (*FileCache) Store

func (c *FileCache) Store(key string, data []byte) error

type Input

type Input struct {
	IP      net.IP
	PTR     string
	Netname string
	Netmail string
	ASN     *int
	ASNName string
	ASNMail string
}

type Match

type Match struct {
	Field   string
	Pattern string
	Weight  int
}

type NetworkCache

type NetworkCache interface {
	Load(key string) (data []byte, updated time.Time, err error)
	Store(key string, data []byte) error
}

type Option

type Option func(*options)

func WithCache

func WithCache(c NetworkCache) Option

func WithCacheDir

func WithCacheDir(dir string) Option

func WithHTTPClient

func WithHTTPClient(c *http.Client) Option

func WithLogger

func WithLogger(l *slog.Logger) Option

type Result

type Result struct {
	ProviderID string
	Name       string
	Category   Category
	Score      int
	Matches    []Match
	OK         bool
}