netident

package module
v1.0.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 27, 2026 License: MIT Imports: 25 Imported by: 0

README

netident

Identify network providers, search engine bots, and cloud platforms by matching IP, PTR, WHOIS, and ASN data against a weighted JSON rule database.

Install

Library:

go get github.com/aredoff/netident

CLI — download a binary from GitHub Releases.

CLI

Identify a provider by IP, PTR, WHOIS, or ASN data:

netident -ip 66.249.79.1 -ptr crawl-66-249-79-1.googlebot.com

Output:

provider=googlebot name="Google Bot" category=bot score=1000
  ptr "*.googlebot.com" weight=1000

More examples:

netident -ip 8.8.8.8 -netname GOOGLE-CLOUD
netident -asn 15169 -asn-name GOOGLE
netident -netmail abuse@amazonaws.com -ptr ec2-1-2-3-4.compute.amazonaws.com
netident -version
netident -validate ./providers.json

Validate a custom config:

netident -validate ./my-providers.json

Output:

config ok: ./my-providers.json

Flags:

Flag Description
-ip IP address
-ptr PTR record
-netname WHOIS netname
-netmail WHOIS netmail
-asn ASN number
-asn-name ASN organization name
-asn-mail ASN abuse email
-validate Validate providers.json, load config, and run a probe identify
-config Path to custom providers.json (default: embedded production config)
-cache-dir Directory for network URL cache (default: $TMPDIR/netident-cache)
-version Print version and exit

At least one input flag is required for identify mode. -validate runs standalone and does not require input flags.

Library usage

package main

import (
    "context"
    "net"

    "github.com/aredoff/netident"
)

func main() {
    det, err := netident.New()
    if err != nil {
        panic(err)
    }

    ctx, cancel := context.WithCancel(context.Background())
    defer cancel()

    if err := det.Start(ctx); err != nil {
        panic(err)
    }
    defer det.Close()

    result := det.Identify(netident.Input{
        IP:  net.ParseIP("66.249.79.1"),
        PTR: "crawl-66-249-79-1.googlebot.com",
    })

    if result.OK && result.Category == netident.CategoryBot {
        // never block bots
        return
    }
}

Lifecycle

Method Description
New* Load JSON config, compile rules, read cache. No goroutines.
Start(ctx) Start background network URL updater.
Close() Stop updater and wait for shutdown.
Identify Match input against all providers, return best result.

Identify works immediately after New using static rules and cached CIDR lists.

Config format

Each provider has weighted rules:

{
  "version": 1,
  "defaults": {"min_score": 1, "rule_weight": 10},
  "providers": [
    {
      "id": "googlebot",
      "name": "Google Bot",
      "category": "bot",
      "rules": {
        "ptr": [{"match": "*.googlebot.com", "weight": 1000}],
        "network_urls": [{
          "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
          "format": "google_prefixes",
          "weight": 900
        }]
      }
    }
  ]
}

All rules use {"match": "...", "weight": N}. Omitted weight defaults to defaults.rule_weight.

Supported rule fields: ptr, networks, network_urls, netname, netmail, asn, asn_name, asn_mail.

Categories: bot, cloud, cdn, hosting, isp, other.

License

MIT

Documentation

Overview

Package netident identifies network providers, bots, and cloud hosts by matching IP, PTR, WHOIS, and ASN fields against a weighted JSON rule database.

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type ASNRule added in v1.0.3 

type ASNRule struct {
	Match  int  `json:"match"`
	Weight *int `json:"weight,omitempty"`
}

type CIDRRule added in v1.0.3 

type CIDRRule struct {
	Match  string `json:"match"`
	Weight *int   `json:"weight,omitempty"`
}

type Category 

type Category string
const (
	CategoryBot     Category = "bot"
	CategoryCloud   Category = "cloud"
	CategoryCDN     Category = "cdn"
	CategoryHosting Category = "hosting"
	CategoryISP     Category = "isp"
	CategoryOther   Category = "other"
)

type Config added in v1.0.3 

type Config struct {
	Version   int        `json:"version"`
	Defaults  Defaults   `json:"defaults"`
	Providers []Provider `json:"providers"`
}

type Defaults added in v1.0.3 

type Defaults struct {
	MinScore   int `json:"min_score"`
	RuleWeight int `json:"rule_weight"`
}

type Detector 

type Detector struct {
	// contains filtered or unexported fields
}

func New 

func New(opts ...Option) (*Detector, error)
Example (Identify) 
package main

import (
	"fmt"

	"github.com/aredoff/netident"
)

func main() {
	d, err := netident.New()
	if err != nil {
		fmt.Println(err)
		return
	}

	result := d.Identify(netident.Input{
		PTR: "crawl-66-249-79-1.googlebot.com",
	})

	fmt.Println(result.ProviderID, result.Score, result.OK)
}

Output:
googlebot 1000 true

func NewFromConfig added in v1.0.3 

func NewFromConfig(cfg Config, opts ...Option) (*Detector, error)

func NewFromFile 

func NewFromFile(path string, opts ...Option) (*Detector, error)

func NewFromJSON 

func NewFromJSON(r io.Reader, opts ...Option) (*Detector, error)

func (*Detector) Close 

func (d *Detector) Close() error

func (*Detector) Identify 

func (d *Detector) Identify(input Input) Result

func (*Detector) IdentifyAll 

func (d *Detector) IdentifyAll(input Input) []Result
Example 
package main

import (
	"fmt"
	"net"

	"github.com/aredoff/netident"
)

func main() {
	d, err := netident.NewFromFile("testdata/providers_minimal.json")
	if err != nil {
		fmt.Println(err)
		return
	}

	ip := net.ParseIP("66.249.79.1")
	all := d.IdentifyAll(netident.Input{IP: ip})
	if len(all) == 0 {
		fmt.Println("none")
		return
	}
	fmt.Println(all[0].ProviderID, all[0].Score)
}

Output:
googlebot 900

func (*Detector) Ready 

func (d *Detector) Ready(ctx context.Context) error

func (*Detector) Start 

func (d *Detector) Start(ctx context.Context) error

type FileCache 

type FileCache struct {
	// contains filtered or unexported fields
}

func NewFileCache 

func NewFileCache(dir string) *FileCache

func (*FileCache) Load 

func (c *FileCache) Load(key string) ([]byte, time.Time, error)

func (*FileCache) Store 

func (c *FileCache) Store(key string, data []byte) error

type Input 

type Input struct {
	IP      net.IP
	PTR     string
	Netname string
	Netmail string
	ASN     *int
	ASNName string
	ASNMail string
}

type Match 

type Match struct {
	Field   string
	Pattern string
	Weight  int
}

type NetworkCache 

type NetworkCache interface {
	Load(key string) (data []byte, updated time.Time, err error)
	Store(key string, data []byte) error
}

type Option 

type Option func(*options)

func WithCache 

func WithCache(c NetworkCache) Option

func WithCacheDir 

func WithCacheDir(dir string) Option

func WithHTTPClient 

func WithHTTPClient(c *http.Client) Option

func WithLogger 

func WithLogger(l *slog.Logger) Option

type Provider added in v1.0.3 

type Provider struct {
	ID       string   `json:"id"`
	Name     string   `json:"name"`
	Category Category `json:"category"`
	Enabled  *bool    `json:"enabled,omitempty"`
	Rules    Rules    `json:"rules"`
}

type Result 

type Result struct {
	ProviderID string
	Name       string
	Category   Category
	Score      int
	Matches    []Match
	OK         bool
}

type Rules added in v1.0.3 

type Rules struct {
	PTR         []StringRule `json:"ptr,omitempty"`
	Networks    []CIDRRule   `json:"networks,omitempty"`
	NetworkURLs []URLSource  `json:"network_urls,omitempty"`
	Netname     []StringRule `json:"netname,omitempty"`
	Netmail     []StringRule `json:"netmail,omitempty"`
	ASN         []ASNRule    `json:"asn,omitempty"`
	ASNName     []StringRule `json:"asn_name,omitempty"`
	ASNMail     []StringRule `json:"asn_mail,omitempty"`
}

type StringRule added in v1.0.3 

type StringRule struct {
	Match  string `json:"match"`
	Weight *int   `json:"weight,omitempty"`
}

type URLSource added in v1.0.3 

type URLSource struct {
	URL            string `json:"url"`
	Format         string `json:"format"`
	Weight         *int   `json:"weight,omitempty"`
	UpdateInterval int    `json:"update_interval,omitempty"`
	Timeout        int    `json:"timeout,omitempty"`
}

Source Files

View all Source files

Directories

Path Synopsis
cmd
netident command
internal
defaultproviders

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.