tldextract

README

tldextract

TLD Extract written in Go. Extracts TLD, domain, and subdomains from a host or URL

Installation

Install tldextract:

go get github.com/mjd2021usa/tldextract

Examples:

Simple example:

  examples/01/main.go

Integration Test example:

  examples/02/main.go

ToDo List:

• Add IPv6 support

Citations and References:

• https://github.com/publicsuffix/list
• https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat
• https://publicsuffix.org/list/public_suffix_list.dat
• https://github.com/joeguo/tldextract
• https://github.com/john-kurkowski/tldextract

Documentation

Index

• Constants
• Variables
• func CreateList(timeout int64) map[string]struct{}
• func CreateNewCacheFile(fqdn string, urls []string, timeout int64) (map[string]struct{}, error)
• func DownloadFile(url string, timeout int64) ([]byte, error)
• func DownloadUrls2List(urls []string, timeout int64) map[string]struct{}
• func GetEnvInt64(envVar string, base int, bitSize int, defaultValue int64) int64
• func GetEnvString(envVar string, defaultValue string) string
• func GetKeys(list map[string]struct{}) []string
• func IsIPv4(ip net.IP) bool
• func IsIPv6(ip net.IP) bool
• func LoadCache(fqdn string, urls []string, refresh bool, timeout int64) (map[string]struct{}, error)
• func LoadCacheFile(fqdn string) (map[string]struct{}, error)
• func NormalizeLines(buffer string) []string
• func ReadFile(fqfn string) ([]byte, error)
• func RemoveNoiseLines(srcLines []string) []string
• func SafeParseInt(str string, base int, bitSize int, defaultValue int64) int64
• func SubDomain(domain string) (string, string)
• func WriteFile(fqfn string, buffer []byte) error
• type Result
• type TLDExtract
  • func New(fqdn string, debug bool) (*TLDExtract, error)
  • func (tlde *TLDExtract) Extract(urlString string) *Result
• type TldNode

Constants

const (
	DefaultCacheTimeout int64 = 10

	// Use raw strings to avoid having to quote the backslashes.
	// \p{Han} allows matching a Chinese, Japanese, and Korean characters
	// \p{Greek} allows matching a Greek characters
	DomainRegexText = `^[a-z0-9-\p{Han}-\p{Greek}]{1,63}$`
	SchemeRegexText = `^([a-z0-9\+\-\.]+:)?//`

	Malformed = iota
	Domain
	IPv4
	IPv6
)

Variables

var (
	DefaultTldUrls = []string{
		"https://publicsuffix.org/list/public_suffix_list.dat",
		"https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat",
	}
)

Functions

func CreateList

func CreateList(timeout int64) map[string]struct{}

CreateList -

func CreateNewCacheFile

func CreateNewCacheFile(fqdn string, urls []string, timeout int64) (map[string]struct{}, error)

CreateNewCacheFile - create new cache file from URLs

func DownloadFile

func DownloadFile(url string, timeout int64) ([]byte, error)

DownloadFile - Get body from URL

func DownloadUrls2List

func DownloadUrls2List(urls []string, timeout int64) map[string]struct{}

DownloadUrls2List - Download N number of URLs and merge the unique rows into a generic key map

func GetEnvInt64

func GetEnvInt64(envVar string, base int, bitSize int, defaultValue int64) int64

func GetEnvString

func GetEnvString(envVar string, defaultValue string) string

func GetKeys

func GetKeys(list map[string]struct{}) []string

GetKeys - get list of keys from generic map as string array

func IsIPv4

func IsIPv4(ip net.IP) bool

func IsIPv6

func IsIPv6(ip net.IP) bool

func LoadCache

func LoadCache(fqdn string, urls []string, refresh bool, timeout int64) (map[string]struct{}, error)

LoadCache - Load cache file with Refresh and fail over options

func LoadCacheFile

func LoadCacheFile(fqdn string) (map[string]struct{}, error)

LoadCacheFile - read cache file and Normalize contents (remove comments, split lines, etc)

func NormalizeLines

func NormalizeLines(buffer string) []string

NormalizeLines - split string into array of strings and remove noise lines

func ReadFile

func ReadFile(fqfn string) ([]byte, error)

ReadFile - Read file into byte array

func RemoveNoiseLines

func RemoveNoiseLines(srcLines []string) []string

RemoveNoiseLines - remove blank lines and comments, converting each kept line to lowercase

func SafeParseInt

func SafeParseInt(str string, base int, bitSize int, defaultValue int64) int64

func SubDomain

func SubDomain(domain string) (string, string)

SubDomain - return sub-domain, domain

func WriteFile

func WriteFile(fqfn string, buffer []byte) error

WriteFile - Write byte array into file

Types

type Result

type Result struct {
	Flag      int
	SubDomain string
	Domain    string
	Tld       string
}

type TLDExtract

type TLDExtract struct {
	CacheTimeout int64
	CacheFile    string
	TldNodes     *TldNode
	Debug        bool
}

func New

func New(fqdn string, debug bool) (*TLDExtract, error)

func (*TLDExtract) Extract

func (tlde *TLDExtract) Extract(urlString string) *Result

type TldNode

type TldNode struct {
	ExceptRule bool
	ValidTld   bool
	// contains filtered or unexported fields
}